2024-09-20T19:18:39.229 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-20T19:18:39.246 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-20T19:18:39.330 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786 branch: wip-adk-testing-2024-09-20-0910 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7912786' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-09-20-0910 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-adk-testing-2024-09-20-0910 sha1: e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 8031 sha1: e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 sleep_before_teardown: 0 subset: 1/20 suite: orch:cephadm suite_branch: wip-adk-testing-2024-09-20-0910 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_e8baca1eac6c3c9c88bf2ea4493e448bd54936a7/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 targets: smithi116.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBECU5S2uLXXiPKMtpBSriD5R0cXI5LU3o9PsR+GphXaaUX/6s7sskFVK//o3Wlq9f1RXEBwNhchE5T6u84QC12M= smithi118.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGNf8NtGpQKoGXWGeVxuNrlTHjQ9ao2ynda4XY75ViJdyp1eBXltIyA6jiYkN981Vcl5/ZSbEdx0yOU4l6EmWqk= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-20_18:25:04 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-20T19:18:39.330 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_e8baca1eac6c3c9c88bf2ea4493e448bd54936a7/qa; will attempt to use it 2024-09-20T19:18:39.331 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_e8baca1eac6c3c9c88bf2ea4493e448bd54936a7/qa/tasks 2024-09-20T19:18:39.331 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-20T19:18:39.332 INFO:teuthology.task.internal:Checking packages... 2024-09-20T19:18:39.351 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'e8baca1eac6c3c9c88bf2ea4493e448bd54936a7' 2024-09-20T19:18:39.352 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-20T19:18:39.352 INFO:teuthology.packaging:ref: None 2024-09-20T19:18:39.352 INFO:teuthology.packaging:tag: None 2024-09-20T19:18:39.352 INFO:teuthology.packaging:branch: wip-adk-testing-2024-09-20-0910 2024-09-20T19:18:39.352 INFO:teuthology.packaging:sha1: e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 2024-09-20T19:18:39.352 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-09-20-0910 2024-09-20T19:18:39.499 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5127.ge8baca1e 2024-09-20T19:18:39.500 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-20T19:18:39.507 INFO:teuthology.task.internal:no buildpackages task found 2024-09-20T19:18:39.507 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-20T19:18:39.512 INFO:teuthology.task.internal:Saving configuration 2024-09-20T19:18:39.526 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-20T19:18:39.533 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-20T19:18:39.558 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi116.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 19:12:54.031843', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBECU5S2uLXXiPKMtpBSriD5R0cXI5LU3o9PsR+GphXaaUX/6s7sskFVK//o3Wlq9f1RXEBwNhchE5T6u84QC12M='} 2024-09-20T19:18:39.578 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi118.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 19:12:54.033844', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGNf8NtGpQKoGXWGeVxuNrlTHjQ9ao2ynda4XY75ViJdyp1eBXltIyA6jiYkN981Vcl5/ZSbEdx0yOU4l6EmWqk='} 2024-09-20T19:18:39.578 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-20T19:18:39.584 INFO:teuthology.task.internal:roles: ubuntu@smithi116.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-20T19:18:39.584 INFO:teuthology.task.internal:roles: ubuntu@smithi118.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-20T19:18:39.584 INFO:teuthology.run_tasks:Running task console_log... 2024-09-20T19:18:39.646 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f96954e8280>, signals=[15]) 2024-09-20T19:18:39.646 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-20T19:18:39.654 INFO:teuthology.task.internal:Opening connections... 2024-09-20T19:18:39.654 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:18:39.655 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi116.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:18:39.733 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi118.front.sepia.ceph.com 2024-09-20T19:18:39.734 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:18:39.804 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-20T19:18:39.811 DEBUG:teuthology.orchestra.run.smithi116:> uname -m 2024-09-20T19:18:39.829 INFO:teuthology.orchestra.run.smithi116.stdout:x86_64 2024-09-20T19:18:39.829 DEBUG:teuthology.orchestra.run.smithi116:> cat /etc/os-release 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:NAME="CentOS Stream" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:VERSION="9" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:ID="centos" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:ID_LIKE="rhel fedora" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:VERSION_ID="9" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:PLATFORM_ID="platform:el9" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:ANSI_COLOR="0;31" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:LOGO="fedora-logo-icon" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T19:18:39.887 INFO:teuthology.orchestra.run.smithi116.stdout:HOME_URL="https://centos.org/" 2024-09-20T19:18:39.888 INFO:teuthology.orchestra.run.smithi116.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T19:18:39.888 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T19:18:39.888 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T19:18:39.888 INFO:teuthology.lock.ops:Updating smithi116.front.sepia.ceph.com on lock server 2024-09-20T19:18:39.924 DEBUG:teuthology.orchestra.run.smithi118:> uname -m 2024-09-20T19:18:39.943 INFO:teuthology.orchestra.run.smithi118.stdout:x86_64 2024-09-20T19:18:39.943 DEBUG:teuthology.orchestra.run.smithi118:> cat /etc/os-release 2024-09-20T19:18:40.001 INFO:teuthology.orchestra.run.smithi118.stdout:NAME="CentOS Stream" 2024-09-20T19:18:40.001 INFO:teuthology.orchestra.run.smithi118.stdout:VERSION="9" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:ID="centos" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:ID_LIKE="rhel fedora" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:VERSION_ID="9" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:PLATFORM_ID="platform:el9" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:ANSI_COLOR="0;31" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:LOGO="fedora-logo-icon" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:HOME_URL="https://centos.org/" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T19:18:40.002 INFO:teuthology.orchestra.run.smithi118.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T19:18:40.003 INFO:teuthology.lock.ops:Updating smithi118.front.sepia.ceph.com on lock server 2024-09-20T19:18:40.024 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-20T19:18:40.032 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-20T19:18:40.038 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-20T19:18:40.038 DEBUG:teuthology.orchestra.run.smithi116:> test '!' -e /home/ubuntu/cephtest 2024-09-20T19:18:40.040 DEBUG:teuthology.orchestra.run.smithi118:> test '!' -e /home/ubuntu/cephtest 2024-09-20T19:18:40.058 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-20T19:18:40.064 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-20T19:18:40.064 DEBUG:teuthology.orchestra.run.smithi116:> test -z $(ls -A /var/lib/ceph) 2024-09-20T19:18:40.097 DEBUG:teuthology.orchestra.run.smithi118:> test -z $(ls -A /var/lib/ceph) 2024-09-20T19:18:40.219 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-20T19:18:40.272 INFO:teuthology.run_tasks:Running task kernel... 2024-09-20T19:18:40.285 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-20T19:18:40.285 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-20T19:18:40.285 DEBUG:teuthology.orchestra.run.smithi116:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T19:18:40.285 DEBUG:teuthology.orchestra.run.smithi118:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T19:18:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:18:40.302 DEBUG:teuthology.orchestra.run.smithi116:> uname -r 2024-09-20T19:18:40.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:18:40.303 DEBUG:teuthology.orchestra.run.smithi118:> uname -r 2024-09-20T19:18:40.360 INFO:teuthology.orchestra.run.smithi116.stdout:5.14.0-503.el9.x86_64 2024-09-20T19:18:40.360 INFO:teuthology.task.kernel:Running kernel on smithi116: 5.14.0-503.el9.x86_64 2024-09-20T19:18:40.360 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum install -y kernel 2024-09-20T19:18:40.361 INFO:teuthology.orchestra.run.smithi118.stdout:5.14.0-503.el9.x86_64 2024-09-20T19:18:40.361 INFO:teuthology.task.kernel:Running kernel on smithi118: 5.14.0-503.el9.x86_64 2024-09-20T19:18:40.361 DEBUG:teuthology.orchestra.run.smithi118:> sudo yum install -y kernel 2024-09-20T19:18:43.334 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - BaseOS 8.7 MB/s | 8.3 MB 00:00 2024-09-20T19:18:43.818 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - BaseOS 4.6 MB/s | 8.3 MB 00:01 2024-09-20T19:18:45.027 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - AppStream 33 MB/s | 20 MB 00:00 2024-09-20T19:18:45.535 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - AppStream 31 MB/s | 20 MB 00:00 2024-09-20T19:18:50.003 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - CRB 9.3 MB/s | 6.5 MB 00:00 2024-09-20T19:18:50.653 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - CRB 3.5 MB/s | 6.5 MB 00:01 2024-09-20T19:18:56.615 INFO:teuthology.orchestra.run.smithi118.stdout:CentOS Stream 9 - Extras packages 4.0 kB/s | 19 kB 00:04 2024-09-20T19:18:57.008 INFO:teuthology.orchestra.run.smithi116.stdout:CentOS Stream 9 - Extras packages 3.4 kB/s | 19 kB 00:05 2024-09-20T19:18:57.476 INFO:teuthology.orchestra.run.smithi118.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-20T19:18:57.792 INFO:teuthology.orchestra.run.smithi116.stdout:Extra Packages for Enterprise Linux 34 MB/s | 23 MB 00:00 2024-09-20T19:19:02.915 INFO:teuthology.orchestra.run.smithi118.stdout:lab-extras 17 kB/s | 1.7 kB 00:00 2024-09-20T19:19:03.240 INFO:teuthology.orchestra.run.smithi116.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-20T19:19:04.813 INFO:teuthology.orchestra.run.smithi118.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T19:19:04.813 INFO:teuthology.orchestra.run.smithi118.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T19:19:04.909 INFO:teuthology.orchestra.run.smithi118.stdout:Dependencies resolved. 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout:================================================================================ 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout: Package Architecture Version Repository Size 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout:================================================================================ 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout:Installing: 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout:Installing dependencies: 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout:Transaction Summary 2024-09-20T19:19:04.920 INFO:teuthology.orchestra.run.smithi118.stdout:================================================================================ 2024-09-20T19:19:04.921 INFO:teuthology.orchestra.run.smithi118.stdout:Install 4 Packages 2024-09-20T19:19:04.921 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:19:04.921 INFO:teuthology.orchestra.run.smithi118.stdout:Total download size: 79 M 2024-09-20T19:19:04.921 INFO:teuthology.orchestra.run.smithi118.stdout:Installed size: 126 M 2024-09-20T19:19:04.921 INFO:teuthology.orchestra.run.smithi118.stdout:Downloading Packages: 2024-09-20T19:19:05.099 INFO:teuthology.orchestra.run.smithi118.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 228 kB/s | 36 kB 00:00 2024-09-20T19:19:05.129 INFO:teuthology.orchestra.run.smithi116.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T19:19:05.130 INFO:teuthology.orchestra.run.smithi116.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T19:19:05.236 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-09-20T19:19:05.245 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-09-20T19:19:05.245 INFO:teuthology.orchestra.run.smithi116.stdout: Package Architecture Version Repository Size 2024-09-20T19:19:05.246 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-09-20T19:19:05.246 INFO:teuthology.orchestra.run.smithi116.stdout:Installing: 2024-09-20T19:19:05.246 INFO:teuthology.orchestra.run.smithi116.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T19:19:05.246 INFO:teuthology.orchestra.run.smithi116.stdout:Installing dependencies: 2024-09-20T19:19:05.246 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T19:19:05.246 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T19:19:05.247 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T19:19:05.247 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:19:05.247 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction Summary 2024-09-20T19:19:05.247 INFO:teuthology.orchestra.run.smithi116.stdout:================================================================================ 2024-09-20T19:19:05.247 INFO:teuthology.orchestra.run.smithi116.stdout:Install 4 Packages 2024-09-20T19:19:05.247 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:19:05.247 INFO:teuthology.orchestra.run.smithi116.stdout:Total download size: 79 M 2024-09-20T19:19:05.248 INFO:teuthology.orchestra.run.smithi116.stdout:Installed size: 126 M 2024-09-20T19:19:05.248 INFO:teuthology.orchestra.run.smithi116.stdout:Downloading Packages: 2024-09-20T19:19:05.716 INFO:teuthology.orchestra.run.smithi116.stdout:(1/4): kernel-core-5.14.0-508.el9.x86_64.rpm 35 MB/s | 16 MB 00:00 2024-09-20T19:19:05.741 INFO:teuthology.orchestra.run.smithi116.stdout:(2/4): kernel-5.14.0-508.el9.x86_64.rpm 76 kB/s | 36 kB 00:00 2024-09-20T19:19:05.783 INFO:teuthology.orchestra.run.smithi118.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 19 MB/s | 16 MB 00:00 2024-09-20T19:19:06.250 INFO:teuthology.orchestra.run.smithi116.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 35 MB/s | 35 MB 00:00 2024-09-20T19:19:06.525 INFO:teuthology.orchestra.run.smithi116.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 35 MB/s | 29 MB 00:00 2024-09-20T19:19:06.526 INFO:teuthology.orchestra.run.smithi116.stdout:-------------------------------------------------------------------------------- 2024-09-20T19:19:06.526 INFO:teuthology.orchestra.run.smithi116.stdout:Total 62 MB/s | 79 MB 00:01 2024-09-20T19:19:06.641 INFO:teuthology.orchestra.run.smithi118.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 20 MB/s | 35 MB 00:01 2024-09-20T19:19:07.204 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction check 2024-09-20T19:19:07.552 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction check succeeded. 2024-09-20T19:19:07.552 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction test 2024-09-20T19:19:08.509 INFO:teuthology.orchestra.run.smithi116.stdout:Transaction test succeeded. 2024-09-20T19:19:08.510 INFO:teuthology.orchestra.run.smithi116.stdout:Running transaction 2024-09-20T19:19:10.948 INFO:teuthology.orchestra.run.smithi116.stdout: Preparing : 1/1 2024-09-20T19:19:11.650 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T19:19:11.734 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T19:19:14.497 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T19:19:14.621 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T19:19:16.641 INFO:teuthology.orchestra.run.smithi118.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 2.5 MB/s | 29 MB 00:11 2024-09-20T19:19:16.642 INFO:teuthology.orchestra.run.smithi118.stdout:-------------------------------------------------------------------------------- 2024-09-20T19:19:16.642 INFO:teuthology.orchestra.run.smithi118.stdout:Total 6.7 MB/s | 79 MB 00:11 2024-09-20T19:19:17.327 INFO:teuthology.orchestra.run.smithi118.stdout:Running transaction check 2024-09-20T19:19:17.677 INFO:teuthology.orchestra.run.smithi118.stdout:Transaction check succeeded. 2024-09-20T19:19:17.678 INFO:teuthology.orchestra.run.smithi118.stdout:Running transaction test 2024-09-20T19:19:18.634 INFO:teuthology.orchestra.run.smithi118.stdout:Transaction test succeeded. 2024-09-20T19:19:18.634 INFO:teuthology.orchestra.run.smithi118.stdout:Running transaction 2024-09-20T19:19:19.451 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T19:19:19.547 INFO:teuthology.orchestra.run.smithi116.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:19:21.082 INFO:teuthology.orchestra.run.smithi118.stdout: Preparing : 1/1 2024-09-20T19:19:21.751 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T19:19:21.840 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T19:19:24.391 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:19:24.592 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T19:19:24.721 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T19:19:29.538 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T19:19:29.672 INFO:teuthology.orchestra.run.smithi118.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:19:34.501 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:02.517 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:02.557 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:02.869 INFO:teuthology.orchestra.run.smithi116.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:02.869 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T19:20:02.869 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T19:20:02.869 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T19:20:03.740 INFO:teuthology.orchestra.run.smithi116.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:03.740 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:20:03.740 INFO:teuthology.orchestra.run.smithi116.stdout:Installed: 2024-09-20T19:20:03.741 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T19:20:03.741 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T19:20:03.741 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T19:20:03.741 INFO:teuthology.orchestra.run.smithi116.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T19:20:03.741 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:20:03.741 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-09-20T19:20:04.105 DEBUG:teuthology.orchestra.run.smithi116:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T19:20:04.192 INFO:teuthology.orchestra.run.smithi116.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T19:20:04.192 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T19:20:04.192 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T19:20:04.192 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T19:20:04.192 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T19:20:04.193 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-20T19:20:04.193 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi116.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T19:20:04.193 DEBUG:teuthology.orchestra.run.smithi116:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T19:20:04.252 INFO:teuthology.orchestra.run.smithi116.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T19:20:04.252 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T19:20:04.252 DEBUG:teuthology.orchestra.run.smithi116:> sudo rpm -qi grub2-tools 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Name : grub2-tools 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Epoch : 1 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Version : 2.06 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Release : 82.el9 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Architecture: x86_64 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Group : Unspecified 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Size : 8274670 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:License : GPLv3+ 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T19:20:04.288 INFO:teuthology.orchestra.run.smithi116.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:Packager : builder@centos.org 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:Vendor : CentOS 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:Summary : Support tools for GRUB. 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:Description : 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:hardware devices. 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:20:04.289 INFO:teuthology.orchestra.run.smithi116.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T19:20:04.290 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T19:20:04.290 INFO:teuthology.task.kernel:Updating grub on smithi116 to boot 5.14.0-508.el9.x86_64 2024-09-20T19:20:04.291 DEBUG:teuthology.orchestra.run.smithi116:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T19:20:04.907 INFO:teuthology.orchestra.run.smithi116.stderr:Generating grub configuration file ... 2024-09-20T19:20:06.203 INFO:teuthology.orchestra.run.smithi116.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T19:20:06.236 INFO:teuthology.orchestra.run.smithi116.stderr:done 2024-09-20T19:20:06.238 DEBUG:teuthology.orchestra.run.smithi116:> mktemp 2024-09-20T19:20:06.284 INFO:teuthology.orchestra.run.smithi116.stdout:/tmp/tmp.phtCoBNyTZ 2024-09-20T19:20:06.284 DEBUG:teuthology.orchestra.run.smithi116:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.phtCoBNyTZ 2024-09-20T19:20:06.311 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod 0666 /tmp/tmp.phtCoBNyTZ 2024-09-20T19:20:06.456 DEBUG:teuthology.orchestra.remote:smithi116:/tmp/tmp.phtCoBNyTZ is 6KB 2024-09-20T19:20:06.503 DEBUG:teuthology.orchestra.run.smithi116:> rm -fr /tmp/tmp.phtCoBNyTZ 2024-09-20T19:20:06.519 DEBUG:teuthology.orchestra.run.smithi116:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T19:20:06.581 INFO:teuthology.orchestra.run.smithi116.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T19:20:06.581 INFO:teuthology.orchestra.run.smithi116.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T19:20:06.581 INFO:teuthology.orchestra.run.smithi116.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T19:20:06.582 INFO:teuthology.orchestra.run.smithi116.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T19:20:06.583 DEBUG:teuthology.orchestra.run.smithi116:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T19:20:06.730 DEBUG:teuthology.orchestra.run.smithi116:> sudo shutdown -r now 2024-09-20T19:20:13.694 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:13.736 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:14.110 INFO:teuthology.orchestra.run.smithi118.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:14.110 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T19:20:14.110 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T19:20:14.110 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T19:20:14.966 INFO:teuthology.orchestra.run.smithi118.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T19:20:14.966 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:20:14.966 INFO:teuthology.orchestra.run.smithi118.stdout:Installed: 2024-09-20T19:20:14.966 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T19:20:14.967 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T19:20:14.967 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T19:20:14.967 INFO:teuthology.orchestra.run.smithi118.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T19:20:14.967 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:20:14.967 INFO:teuthology.orchestra.run.smithi118.stdout:Complete! 2024-09-20T19:20:15.378 DEBUG:teuthology.orchestra.run.smithi118:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T19:20:15.462 INFO:teuthology.orchestra.run.smithi118.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T19:20:15.462 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T19:20:15.462 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T19:20:15.462 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T19:20:15.463 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T19:20:15.463 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-20T19:20:15.463 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi118.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T19:20:15.463 DEBUG:teuthology.orchestra.run.smithi118:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T19:20:15.520 INFO:teuthology.orchestra.run.smithi118.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T19:20:15.521 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T19:20:15.521 DEBUG:teuthology.orchestra.run.smithi118:> sudo rpm -qi grub2-tools 2024-09-20T19:20:15.552 INFO:teuthology.orchestra.run.smithi118.stdout:Name : grub2-tools 2024-09-20T19:20:15.552 INFO:teuthology.orchestra.run.smithi118.stdout:Epoch : 1 2024-09-20T19:20:15.552 INFO:teuthology.orchestra.run.smithi118.stdout:Version : 2.06 2024-09-20T19:20:15.552 INFO:teuthology.orchestra.run.smithi118.stdout:Release : 82.el9 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Architecture: x86_64 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Group : Unspecified 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Size : 8274670 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:License : GPLv3+ 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Packager : builder@centos.org 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Vendor : CentOS 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Summary : Support tools for GRUB. 2024-09-20T19:20:15.553 INFO:teuthology.orchestra.run.smithi118.stdout:Description : 2024-09-20T19:20:15.554 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:20:15.554 INFO:teuthology.orchestra.run.smithi118.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T19:20:15.554 INFO:teuthology.orchestra.run.smithi118.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T19:20:15.554 INFO:teuthology.orchestra.run.smithi118.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T19:20:15.554 INFO:teuthology.orchestra.run.smithi118.stdout:hardware devices. 2024-09-20T19:20:15.554 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:20:15.554 INFO:teuthology.orchestra.run.smithi118.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T19:20:15.555 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T19:20:15.555 INFO:teuthology.task.kernel:Updating grub on smithi118 to boot 5.14.0-508.el9.x86_64 2024-09-20T19:20:15.555 DEBUG:teuthology.orchestra.run.smithi118:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T19:20:16.244 INFO:teuthology.orchestra.run.smithi118.stderr:Generating grub configuration file ... 2024-09-20T19:20:17.584 INFO:teuthology.orchestra.run.smithi118.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T19:20:17.615 INFO:teuthology.orchestra.run.smithi118.stderr:done 2024-09-20T19:20:17.618 DEBUG:teuthology.orchestra.run.smithi118:> mktemp 2024-09-20T19:20:17.633 INFO:teuthology.orchestra.run.smithi118.stdout:/tmp/tmp.uWnMO0daRT 2024-09-20T19:20:17.633 DEBUG:teuthology.orchestra.run.smithi118:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.uWnMO0daRT 2024-09-20T19:20:17.699 DEBUG:teuthology.orchestra.run.smithi118:> sudo chmod 0666 /tmp/tmp.uWnMO0daRT 2024-09-20T19:20:17.845 DEBUG:teuthology.orchestra.remote:smithi118:/tmp/tmp.uWnMO0daRT is 6KB 2024-09-20T19:20:17.864 DEBUG:teuthology.orchestra.run.smithi118:> rm -fr /tmp/tmp.uWnMO0daRT 2024-09-20T19:20:17.878 DEBUG:teuthology.orchestra.run.smithi118:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T19:20:17.941 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T19:20:17.941 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T19:20:17.941 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T19:20:17.941 INFO:teuthology.orchestra.run.smithi118.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T19:20:17.942 DEBUG:teuthology.orchestra.run.smithi118:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T19:20:18.092 DEBUG:teuthology.orchestra.run.smithi118:> sudo shutdown -r now 2024-09-20T19:20:36.742 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-20T19:20:36.743 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi116.front.sepia.ceph.com' 2024-09-20T19:20:36.743 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi116.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:20:48.106 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-20T19:20:48.106 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-20T19:20:48.107 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:21:22.671 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.118 2024-09-20T19:21:31.680 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-20T19:21:31.680 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:21:34.735 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.118 2024-09-20T19:21:36.746 DEBUG:teuthology.orchestra.remote:timed out 2024-09-20T19:21:45.756 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi116.front.sepia.ceph.com' 2024-09-20T19:21:45.757 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi116.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:21:46.205 DEBUG:teuthology.orchestra.run.smithi116:> true 2024-09-20T19:21:46.622 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi116.front.sepia.ceph.com' 2024-09-20T19:21:46.622 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-20T19:21:46.622 DEBUG:teuthology.orchestra.run.smithi116:> uname -r 2024-09-20T19:21:46.677 INFO:teuthology.orchestra.run.smithi116.stdout:5.14.0-508.el9.x86_64 2024-09-20T19:21:46.677 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T19:21:46.677 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T19:21:46.677 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T19:21:46.735 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-20T19:21:46.736 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:21:47.678 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-20T19:21:47.678 DEBUG:teuthology.orchestra.run.smithi116:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T19:21:47.756 INFO:teuthology.orchestra.run.smithi116.stdout:ttyS1 2024-09-20T19:21:47.780 DEBUG:teuthology.parallel:result is None 2024-09-20T19:22:18.643 DEBUG:teuthology.orchestra.run.smithi118:> true 2024-09-20T19:22:19.083 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-20T19:22:19.083 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-20T19:22:19.084 DEBUG:teuthology.orchestra.run.smithi118:> uname -r 2024-09-20T19:22:19.098 INFO:teuthology.orchestra.run.smithi118.stdout:5.14.0-508.el9.x86_64 2024-09-20T19:22:19.098 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T19:22:19.098 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T19:22:19.098 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T19:22:20.099 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-20T19:22:20.099 DEBUG:teuthology.orchestra.run.smithi118:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T19:22:20.218 INFO:teuthology.orchestra.run.smithi118.stdout:ttyS1 2024-09-20T19:22:20.243 DEBUG:teuthology.parallel:result is None 2024-09-20T19:22:20.243 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-20T19:22:20.257 INFO:teuthology.task.internal:Creating test directory... 2024-09-20T19:22:20.257 DEBUG:teuthology.orchestra.run.smithi116:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T19:22:20.260 DEBUG:teuthology.orchestra.run.smithi118:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T19:22:20.304 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-20T19:22:20.313 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-20T19:22:20.392 INFO:teuthology.task.internal:Creating archive directory... 2024-09-20T19:22:20.393 DEBUG:teuthology.orchestra.run.smithi116:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T19:22:20.395 DEBUG:teuthology.orchestra.run.smithi118:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T19:22:20.456 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-20T19:22:20.462 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-20T19:22:20.463 DEBUG:teuthology.orchestra.run.smithi116:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T19:22:20.465 DEBUG:teuthology.orchestra.run.smithi118:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T19:22:20.495 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T19:22:20.505 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T19:22:20.506 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T19:22:20.516 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T19:22:20.517 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-20T19:22:20.524 INFO:teuthology.task.internal:Configuring sudo... 2024-09-20T19:22:20.524 DEBUG:teuthology.orchestra.run.smithi116:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T19:22:20.550 DEBUG:teuthology.orchestra.run.smithi118:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T19:22:20.583 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-20T19:22:20.592 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-20T19:22:20.592 DEBUG:teuthology.orchestra.run.smithi116:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T19:22:20.617 DEBUG:teuthology.orchestra.run.smithi118:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T19:22:20.639 DEBUG:teuthology.orchestra.run.smithi116:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T19:22:20.714 DEBUG:teuthology.orchestra.run.smithi116:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T19:22:20.759 DEBUG:teuthology.orchestra.run.smithi116:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T19:22:20.841 DEBUG:teuthology.orchestra.run.smithi116:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T19:22:20.883 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:22:20.883 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T19:22:20.948 DEBUG:teuthology.orchestra.run.smithi118:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T19:22:20.991 DEBUG:teuthology.orchestra.run.smithi118:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T19:22:21.119 DEBUG:teuthology.orchestra.run.smithi118:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T19:22:21.164 DEBUG:teuthology.orchestra.run.smithi118:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T19:22:21.204 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-20T19:22:21.204 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T19:22:21.268 DEBUG:teuthology.orchestra.run.smithi116:> sudo service rsyslog restart 2024-09-20T19:22:21.270 DEBUG:teuthology.orchestra.run.smithi118:> sudo service rsyslog restart 2024-09-20T19:22:21.322 INFO:teuthology.orchestra.run.smithi116.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T19:22:21.351 INFO:teuthology.orchestra.run.smithi118.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T19:22:21.726 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-20T19:22:21.733 INFO:teuthology.task.internal:Starting timer... 2024-09-20T19:22:21.733 INFO:teuthology.run_tasks:Running task pcp... 2024-09-20T19:22:21.743 INFO:teuthology.run_tasks:Running task selinux... 2024-09-20T19:22:21.751 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-20T19:22:21.751 DEBUG:teuthology.orchestra.run.smithi116:> sudo service auditd rotate 2024-09-20T19:22:21.813 INFO:teuthology.orchestra.run.smithi116.stdout:Rotating logs: 2024-09-20T19:22:21.815 DEBUG:teuthology.orchestra.run.smithi118:> sudo service auditd rotate 2024-09-20T19:22:21.860 INFO:teuthology.orchestra.run.smithi118.stdout:Rotating logs: 2024-09-20T19:22:21.862 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-20T19:22:21.862 DEBUG:teuthology.orchestra.run.smithi116:> /usr/sbin/getenforce 2024-09-20T19:22:21.889 INFO:teuthology.orchestra.run.smithi116.stdout:Permissive 2024-09-20T19:22:21.889 DEBUG:teuthology.orchestra.run.smithi118:> /usr/sbin/getenforce 2024-09-20T19:22:21.922 INFO:teuthology.orchestra.run.smithi118.stdout:Permissive 2024-09-20T19:22:21.922 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi116.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi118.front.sepia.ceph.com': 'permissive'} 2024-09-20T19:22:21.922 DEBUG:teuthology.orchestra.run.smithi116:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T19:22:21.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:22:21.954 DEBUG:teuthology.orchestra.run.smithi118:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T19:22:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:22:21.987 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-20T19:22:21.987 DEBUG:teuthology.orchestra.run.smithi116:> sudo /usr/sbin/setenforce permissive 2024-09-20T19:22:22.027 DEBUG:teuthology.orchestra.run.smithi118:> sudo /usr/sbin/setenforce permissive 2024-09-20T19:22:22.058 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-20T19:22:22.067 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-20T19:22:22.070 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-20T19:22:22.186 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-20T19:22:22.195 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-20T19:22:22.196 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi116.front.sepia.ceph.com,smithi118.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-20T19:26:03.156 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi116.front.sepia.ceph.com'), Remote(name='ubuntu@smithi118.front.sepia.ceph.com')] 2024-09-20T19:26:03.157 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi116.front.sepia.ceph.com' 2024-09-20T19:26:03.158 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi116.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:26:03.234 DEBUG:teuthology.orchestra.run.smithi116:> true 2024-09-20T19:26:03.318 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi116.front.sepia.ceph.com' 2024-09-20T19:26:03.318 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-20T19:26:03.319 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi118.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T19:26:03.401 DEBUG:teuthology.orchestra.run.smithi118:> true 2024-09-20T19:26:03.490 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi118.front.sepia.ceph.com' 2024-09-20T19:26:03.490 INFO:teuthology.run_tasks:Running task clock... 2024-09-20T19:26:03.504 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-20T19:26:03.504 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T19:26:03.504 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T19:26:03.506 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T19:26:03.507 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T19:26:03.533 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T19:26:03.547 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T19:26:03.560 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T19:26:03.570 INFO:teuthology.orchestra.run.smithi116.stderr:sudo: ntpd: command not found 2024-09-20T19:26:03.574 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T19:26:03.582 INFO:teuthology.orchestra.run.smithi116.stdout:506 Cannot talk to daemon 2024-09-20T19:26:03.596 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T19:26:03.598 INFO:teuthology.orchestra.run.smithi118.stderr:sudo: ntpd: command not found 2024-09-20T19:26:03.609 INFO:teuthology.orchestra.run.smithi118.stdout:506 Cannot talk to daemon 2024-09-20T19:26:03.610 INFO:teuthology.orchestra.run.smithi116.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T19:26:03.623 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T19:26:03.637 INFO:teuthology.orchestra.run.smithi118.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T19:26:03.662 INFO:teuthology.orchestra.run.smithi116.stderr:bash: line 1: ntpq: command not found 2024-09-20T19:26:03.666 INFO:teuthology.orchestra.run.smithi116.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T19:26:03.666 INFO:teuthology.orchestra.run.smithi116.stdout:=============================================================================== 2024-09-20T19:26:03.666 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.666 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.667 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.667 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.695 INFO:teuthology.orchestra.run.smithi118.stderr:bash: line 1: ntpq: command not found 2024-09-20T19:26:03.698 INFO:teuthology.orchestra.run.smithi118.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T19:26:03.698 INFO:teuthology.orchestra.run.smithi118.stdout:=============================================================================== 2024-09-20T19:26:03.698 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.698 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.698 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.699 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:26:03.700 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-20T19:26:03.816 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'e8baca1eac6c3c9c88bf2ea4493e448bd54936a7'} 2024-09-20T19:26:03.816 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-20T19:26:03.817 INFO:tasks.cephadm:Cluster fsid is 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:26:03.817 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-20T19:26:03.817 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-20T19:26:03.818 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi116': '172.21.15.116', 'mon.smithi118': '172.21.15.118'} 2024-09-20T19:26:03.818 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-20T19:26:03.818 DEBUG:teuthology.orchestra.run.smithi116:> sudo hostname $(hostname -s) 2024-09-20T19:26:03.845 DEBUG:teuthology.orchestra.run.smithi118:> sudo hostname $(hostname -s) 2024-09-20T19:26:03.872 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-20T19:26:03.872 DEBUG:teuthology.orchestra.run.smithi116:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T19:26:04.153 INFO:teuthology.orchestra.run.smithi116.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 19:26 /home/ubuntu/cephtest/cephadm 2024-09-20T19:26:04.153 DEBUG:teuthology.orchestra.run.smithi118:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T19:26:04.290 INFO:teuthology.orchestra.run.smithi118.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 19:26 /home/ubuntu/cephtest/cephadm 2024-09-20T19:26:04.290 DEBUG:teuthology.orchestra.run.smithi116:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T19:26:04.307 DEBUG:teuthology.orchestra.run.smithi118:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T19:26:04.333 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-20T19:26:04.333 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T19:26:04.354 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T19:26:05.861 INFO:teuthology.orchestra.run.smithi116.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T19:26:06.126 INFO:teuthology.orchestra.run.smithi118.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T19:26:17.549 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:26:17.549 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T19:26:17.550 INFO:teuthology.orchestra.run.smithi116.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T19:26:17.550 INFO:teuthology.orchestra.run.smithi116.stdout: "repo_digests": [ 2024-09-20T19:26:17.550 INFO:teuthology.orchestra.run.smithi116.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T19:26:17.550 INFO:teuthology.orchestra.run.smithi116.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T19:26:17.550 INFO:teuthology.orchestra.run.smithi116.stdout: ] 2024-09-20T19:26:17.550 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:27:09.007 INFO:teuthology.orchestra.run.smithi118.stdout:{ 2024-09-20T19:27:09.008 INFO:teuthology.orchestra.run.smithi118.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T19:27:09.008 INFO:teuthology.orchestra.run.smithi118.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T19:27:09.008 INFO:teuthology.orchestra.run.smithi118.stdout: "repo_digests": [ 2024-09-20T19:27:09.008 INFO:teuthology.orchestra.run.smithi118.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T19:27:09.008 INFO:teuthology.orchestra.run.smithi118.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T19:27:09.008 INFO:teuthology.orchestra.run.smithi118.stdout: ] 2024-09-20T19:27:09.008 INFO:teuthology.orchestra.run.smithi118.stdout:} 2024-09-20T19:27:09.027 DEBUG:teuthology.orchestra.run.smithi116:> sudo mkdir -p /etc/ceph 2024-09-20T19:27:09.061 DEBUG:teuthology.orchestra.run.smithi118:> sudo mkdir -p /etc/ceph 2024-09-20T19:27:09.089 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod 777 /etc/ceph 2024-09-20T19:27:09.130 DEBUG:teuthology.orchestra.run.smithi118:> sudo chmod 777 /etc/ceph 2024-09-20T19:27:09.155 INFO:tasks.cephadm:Writing seed config... 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-20T19:27:09.156 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-20T19:27:09.157 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:27:09.157 DEBUG:teuthology.orchestra.run.smithi116:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-20T19:27:09.187 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2d2015e0-7786-11ef-baf5-efdc52797490 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-20T19:27:09.188 DEBUG:teuthology.orchestra.run.smithi116:mon.smithi116> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service 2024-09-20T19:27:09.229 INFO:tasks.cephadm:Bootstrapping... 2024-09-20T19:27:09.230 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.116 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-20T19:27:09.422 INFO:teuthology.orchestra.run.smithi116.stderr:-------------------------------------------------------------------------------- 2024-09-20T19:27:09.422 INFO:teuthology.orchestra.run.smithi116.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '2d2015e0-7786-11ef-baf5-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.116', '--skip-admin-label'] 2024-09-20T19:27:09.444 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: 5.2.2 2024-09-20T19:27:09.447 INFO:teuthology.orchestra.run.smithi116.stderr:Verifying podman|docker is present... 2024-09-20T19:27:09.467 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: 5.2.2 2024-09-20T19:27:09.470 INFO:teuthology.orchestra.run.smithi116.stderr:Verifying lvm2 is present... 2024-09-20T19:27:09.470 INFO:teuthology.orchestra.run.smithi116.stderr:Verifying time synchronization is in place... 2024-09-20T19:27:09.477 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T19:27:09.483 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: inactive 2024-09-20T19:27:09.490 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: enabled 2024-09-20T19:27:09.496 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: active 2024-09-20T19:27:09.497 INFO:teuthology.orchestra.run.smithi116.stderr:Unit chronyd.service is enabled and running 2024-09-20T19:27:09.497 INFO:teuthology.orchestra.run.smithi116.stderr:Repeating the final host check... 2024-09-20T19:27:09.517 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: 5.2.2 2024-09-20T19:27:09.519 INFO:teuthology.orchestra.run.smithi116.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-20T19:27:09.519 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl is present 2024-09-20T19:27:09.519 INFO:teuthology.orchestra.run.smithi116.stderr:lvcreate is present 2024-09-20T19:27:09.525 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T19:27:09.531 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: inactive 2024-09-20T19:27:09.538 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: enabled 2024-09-20T19:27:09.544 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: active 2024-09-20T19:27:09.545 INFO:teuthology.orchestra.run.smithi116.stderr:Unit chronyd.service is enabled and running 2024-09-20T19:27:09.545 INFO:teuthology.orchestra.run.smithi116.stderr:Host looks OK 2024-09-20T19:27:09.545 INFO:teuthology.orchestra.run.smithi116.stderr:Cluster fsid: 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:09.545 INFO:teuthology.orchestra.run.smithi116.stderr:Acquiring lock 139663998494608 on /run/cephadm/2d2015e0-7786-11ef-baf5-efdc52797490.lock 2024-09-20T19:27:09.545 INFO:teuthology.orchestra.run.smithi116.stderr:Lock 139663998494608 acquired on /run/cephadm/2d2015e0-7786-11ef-baf5-efdc52797490.lock 2024-09-20T19:27:09.546 INFO:teuthology.orchestra.run.smithi116.stderr:Verifying IP 172.21.15.116 port 3300 ... 2024-09-20T19:27:09.546 INFO:teuthology.orchestra.run.smithi116.stderr:Verifying IP 172.21.15.116 port 6789 ... 2024-09-20T19:27:09.546 INFO:teuthology.orchestra.run.smithi116.stderr:Base mon IP is 172.21.15.116, final addrv is [v2:172.21.15.116:3300,v1:172.21.15.116:6789] 2024-09-20T19:27:09.548 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.116 metric 100 2024-09-20T19:27:09.548 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.116 metric 100 2024-09-20T19:27:09.551 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-20T19:27:09.551 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-20T19:27:09.551 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-20T19:27:09.554 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-20T19:27:09.554 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-20T19:27:09.554 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T19:27:09.554 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-20T19:27:09.554 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:3f67/64 scope link noprefixroute 2024-09-20T19:27:09.554 INFO:teuthology.orchestra.run.smithi116.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T19:27:09.555 INFO:teuthology.orchestra.run.smithi116.stderr:Mon IP `172.21.15.116` is in CIDR network `172.21.0.0/20` 2024-09-20T19:27:09.555 INFO:teuthology.orchestra.run.smithi116.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-20T19:27:09.555 INFO:teuthology.orchestra.run.smithi116.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T19:27:09.656 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-20T19:27:10.062 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Getting image source signatures 2024-09-20T19:27:10.063 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-20T19:27:10.063 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-20T19:27:10.063 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-20T19:27:10.063 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-20T19:27:10.063 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-20T19:27:10.063 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T19:27:10.209 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: Writing manifest to image destination 2024-09-20T19:27:10.329 INFO:teuthology.orchestra.run.smithi116.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T19:27:10.719 INFO:teuthology.orchestra.run.smithi116.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T19:27:11.228 INFO:teuthology.orchestra.run.smithi116.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T19:27:11.228 INFO:teuthology.orchestra.run.smithi116.stderr:Extracting ceph user uid/gid from container image... 2024-09-20T19:27:11.636 INFO:teuthology.orchestra.run.smithi116.stderr:stat: 167 167 2024-09-20T19:27:12.116 INFO:teuthology.orchestra.run.smithi116.stderr:Creating initial keys... 2024-09-20T19:27:12.516 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph-authtool: AQCQzO1mX2HJGRAA7FTukrZXzOUhvKcon3ZAFA== 2024-09-20T19:27:13.442 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph-authtool: AQCRzO1mDBQ+FxAAGRbbpUty0ZU3EKIFUe8vtg== 2024-09-20T19:27:14.452 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph-authtool: AQCSzO1mhuqtFhAA9B/dasbORRQl7wUWqFGUZw== 2024-09-20T19:27:14.943 INFO:teuthology.orchestra.run.smithi116.stderr:Creating initial monmap... 2024-09-20T19:27:15.383 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T19:27:15.383 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-20T19:27:15.383 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:15.383 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T19:27:15.931 INFO:teuthology.orchestra.run.smithi116.stderr:monmaptool for smithi116 [v2:172.21.15.116:3300,v1:172.21.15.116:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T19:27:15.931 INFO:teuthology.orchestra.run.smithi116.stderr:setting min_mon_release = octopus 2024-09-20T19:27:15.931 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/monmaptool: set fsid to 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:15.931 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T19:27:15.931 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:27:15.932 INFO:teuthology.orchestra.run.smithi116.stderr:Creating mon... 2024-09-20T19:27:16.870 INFO:teuthology.orchestra.run.smithi116.stderr:create mon.smithi116 on 2024-09-20T19:27:17.089 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-20T19:27:17.240 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2d2015e0-7786-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-2d2015e0-7786-11ef-baf5-efdc52797490.target. 2024-09-20T19:27:17.240 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-2d2015e0-7786-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-2d2015e0-7786-11ef-baf5-efdc52797490.target. 2024-09-20T19:27:17.574 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Failed to reset failed state of unit ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service: Unit ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service not loaded. 2024-09-20T19:27:17.582 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Created symlink /etc/systemd/system/ceph-2d2015e0-7786-11ef-baf5-efdc52797490.target.wants/ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service → /etc/systemd/system/ceph-2d2015e0-7786-11ef-baf5-efdc52797490@.service. 2024-09-20T19:27:18.237 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: disabled 2024-09-20T19:27:18.244 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: inactive 2024-09-20T19:27:18.244 INFO:teuthology.orchestra.run.smithi116.stderr:firewalld.service is not enabled 2024-09-20T19:27:18.244 INFO:teuthology.orchestra.run.smithi116.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T19:27:18.245 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for mon to start... 2024-09-20T19:27:18.245 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for mon... 2024-09-20T19:27:18.798 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: cluster: 2024-09-20T19:27:18.798 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: id: 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:18.798 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-20T19:27:18.798 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:18.798 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: services: 2024-09-20T19:27:18.798 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi116 (age 0.615939s) 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: data: 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: pgs: 2024-09-20T19:27:18.799 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:19.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:19 smithi116 ceph-mon[23369]: from='client.? 172.21.15.116:0/1448617069' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-20T19:27:19.381 INFO:teuthology.orchestra.run.smithi116.stderr:mon is available 2024-09-20T19:27:19.381 INFO:teuthology.orchestra.run.smithi116.stderr:Assimilating anything we can from ceph.conf... 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: [global] 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: fsid = 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.116:3300,v1:172.21.15.116:6789] 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T19:27:19.862 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T19:27:19.863 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:19.863 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: [mgr] 2024-09-20T19:27:19.863 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T19:27:19.863 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:19.863 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: [osd] 2024-09-20T19:27:19.863 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T19:27:19.863 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T19:27:20.404 INFO:teuthology.orchestra.run.smithi116.stderr:Generating new minimal ceph.conf... 2024-09-20T19:27:20.525 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:20 smithi116 ceph-mon[23369]: from='client.? 172.21.15.116:0/2262400878' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T19:27:20.525 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:20 smithi116 ceph-mon[23369]: from='client.? 172.21.15.116:0/2262400878' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-20T19:27:21.483 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:21 smithi116 ceph-mon[23369]: from='client.? 172.21.15.116:0/706897620' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:27:21.485 INFO:teuthology.orchestra.run.smithi116.stderr:Restarting the monitor... 2024-09-20T19:27:21.870 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:21 smithi116 systemd[1]: Stopping Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:27:21.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:21 smithi116 bash[23589]: Error: no container with name or ID "ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon.smithi116" found: no such container 2024-09-20T19:27:21.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:21 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[23365]: 2024-09-20T19:27:21.687+0000 7fc50e311700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi116 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:27:21.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:21 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[23365]: 2024-09-20T19:27:21.687+0000 7fc50e311700 -1 mon.smithi116@0(leader) e1 *** Got Signal Terminated *** 2024-09-20T19:27:22.145 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:21 smithi116 podman[23600]: 2024-09-20 19:27:21.892598285 +0000 UTC m=+0.339526851 container died 9c9e92946042e804da91f85b1fc0b24977ce1c5c8c2c24d139c58d12d9cee79a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, vendor=Red Hat, Inc., distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, ceph=True, release=754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, architecture=x86_64, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870) 2024-09-20T19:27:22.520 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 podman[23600]: 2024-09-20 19:27:22.236923566 +0000 UTC m=+0.683852166 container cleanup 9c9e92946042e804da91f85b1fc0b24977ce1c5c8c2c24d139c58d12d9cee79a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, version=8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, RELEASE=HEAD, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream) 2024-09-20T19:27:22.520 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 bash[23600]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116 2024-09-20T19:27:22.520 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 bash[23627]: Error: no container with name or ID "ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon.smithi116" found: no such container 2024-09-20T19:27:22.831 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 podman[23612]: 2024-09-20 19:27:22.521180338 +0000 UTC m=+0.626289979 container remove 9c9e92946042e804da91f85b1fc0b24977ce1c5c8c2c24d139c58d12d9cee79a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_CLEAN=True, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-20T19:27:22.831 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service: Deactivated successfully. 2024-09-20T19:27:22.831 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 systemd[1]: Stopped Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:27:22.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 systemd[1]: Starting Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:27:22.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 podman[23694]: 2024-09-20 19:27:22.746867106 +0000 UTC m=+0.106892258 container create 3a5bddbb825d1e7ef94008d2eacfb11e06fc138b7f333c0ba117d297da8f8fa3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, version=8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , distribution-scope=public, release=754, ceph=True, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-type=git, vendor=Red Hat, Inc.) 2024-09-20T19:27:22.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 podman[23694]: 2024-09-20 19:27:22.656963932 +0000 UTC m=+0.016989049 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-20T19:27:23.018 INFO:teuthology.orchestra.run.smithi116.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-20T19:27:23.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 podman[23694]: 2024-09-20 19:27:22.932626162 +0000 UTC m=+0.292651288 container init 3a5bddbb825d1e7ef94008d2eacfb11e06fc138b7f333c0ba117d297da8f8fa3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, build-date=2022-05-03T08:36:31.336870, vcs-type=git, RELEASE=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, name=centos-stream, architecture=x86_64, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , release=754, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-20T19:27:23.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 podman[23694]: 2024-09-20 19:27:22.936466108 +0000 UTC m=+0.296491224 container start 3a5bddbb825d1e7ef94008d2eacfb11e06fc138b7f333c0ba117d297da8f8fa3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, distribution-scope=public, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, release=754, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-20T19:27:23.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T19:27:23.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-20T19:27:23.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: pidfile_write: ignore empty --pid-file 2024-09-20T19:27:23.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: load: jerasure load: lrc 2024-09-20T19:27:23.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: RocksDB version: 6.15.5 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Compile date Apr 18 2022 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: DB SUMMARY 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: DB Session ID: I9YVA5NI9V4UHBX02G0T 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: CURRENT file: CURRENT 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi116/store.db dir, Total Num: 1, files: 000008.sst 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi116/store.db: 000010.log size: 82739 ; 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.error_if_exists: 0 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.create_if_missing: 0 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.paranoid_checks: 1 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T19:27:23.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.env: 0x5648932c6860 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.fs: Posix File System 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.info_log: 0x5648948d5fa0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.statistics: (nil) 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.use_fsync: 0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_log_file_size: 0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.allow_fallocate: 1 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.use_direct_reads: 0 2024-09-20T19:27:23.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.db_log_dir: 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi116/store.db 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.write_buffer_manager: 0x5648949c62d0 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-20T19:27:23.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.unordered_write: 0 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.row_cache: None 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.wal_filter: None 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T19:27:23.248 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.preserve_deletes: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.two_write_queues: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.atomic_flush: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.log_readahead_size: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_background_jobs: 2 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_background_compactions: -1 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_subcompactions: 1 2024-09-20T19:27:23.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T19:27:23.250 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T19:27:23.250 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_open_files: -1 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_background_flushes: -1 2024-09-20T19:27:23.251 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Compression algorithms supported: 2024-09-20T19:27:23.252 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T19:27:23.252 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kZSTD supported: 0 2024-09-20T19:27:23.252 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kXpressCompression supported: 0 2024-09-20T19:27:23.252 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T19:27:23.252 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kLZ4Compression supported: 1 2024-09-20T19:27:23.252 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kBZip2Compression supported: 0 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kZlibCompression supported: 1 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: kSnappyCompression supported: 1 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi116/store.db/MANIFEST-000009 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.merge_operator: 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_filter: None 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T19:27:23.254 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5648948a1d00) 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: cache_index_and_filter_blocks: 1 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: pin_top_level_index_and_filter: 1 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: index_type: 0 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: data_block_index_type: 0 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: index_shortening: 1 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: hash_index_allow_collision: 1 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: checksum: 1 2024-09-20T19:27:23.255 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: no_block_cache: 0 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache: 0x56489490c170 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache_name: BinnedLRUCache 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache_options: 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: capacity : 536870912 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: num_shard_bits : 4 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: strict_capacity_limit : 0 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: high_pri_pool_ratio: 0.000 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache_compressed: (nil) 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: persistent_cache: (nil) 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_size: 4096 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_size_deviation: 10 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_restart_interval: 16 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: index_block_restart_interval: 1 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: metadata_block_size: 4096 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: partition_filters: 0 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: use_delta_encoding: 1 2024-09-20T19:27:23.256 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: whole_key_filtering: 1 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: verify_compression: 0 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: read_amp_bytes_per_bit: 0 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: format_version: 4 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: enable_index_compression: 1 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_align: 0 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression: NoCompression 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.num_levels: 7 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T19:27:23.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T19:27:23.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T19:27:23.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.arena_block_size: 4194304 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T19:27:23.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.table_properties_collectors: 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.inplace_update_support: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.bloom_locality: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.max_successive_merges: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T19:27:23.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.ttl: 2592000 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.enable_blob_files: false 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.min_blob_size: 0 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi116/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726860442962531, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-20T19:27:23.262 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726860442963283, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79421, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77685, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71994, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726860442, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "2e5918d5-f508-48da-99c1-2ef25034ac58", "db_session_id": "I9YVA5NI9V4UHBX02G0T"}} 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726860442963630, "job": 1, "event": "recovery_finished"} 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi116/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5648948f2a80 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: DB pointer 0x564894902000 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** DB Stats ** 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-20T19:27:23.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** Compaction Stats [default] ** 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.00 0.00 1 0.001 0 0 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.00 0.00 1 0.001 0 0 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.00 0.00 1 0.001 0 0 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** Compaction Stats [default] ** 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 142.6 0.00 0.00 1 0.001 0 0 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T19:27:23.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative compaction: 0.00 GB write, 24.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval compaction: 0.00 GB write, 24.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** Compaction Stats [default] ** 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.00 0.00 1 0.001 0 0 2024-09-20T19:27:23.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.6 0.00 0.00 1 0.001 0 0 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** Compaction Stats [default] ** 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 142.6 0.00 0.00 1 0.001 0 0 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative compaction: 0.00 GB write, 24.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:27:23.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: starting mon.smithi116 rank 0 at public addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] at bind addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi116 fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???) e1 preinit fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???).mds e1 new map 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???).mds e1 print_map 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: e1 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: legacy client fscid: -1 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: No filesystems configured 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T19:27:23.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: mon.smithi116@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expand map: {default=false} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta from 'false' to 'false' 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expanded map: {default=false} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expand map: {default=info} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta from 'info' to 'info' 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expanded map: {default=info} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expand map: {default=daemon} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expanded map: {default=daemon} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expand map: {default=debug} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta from 'debug' to 'debug' 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:22 smithi116 ceph-mon[23713]: expand_channel_meta expanded map: {default=debug} 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:23 smithi116 bash[23694]: 3a5bddbb825d1e7ef94008d2eacfb11e06fc138b7f333c0ba117d297da8f8fa3 2024-09-20T19:27:23.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:23 smithi116 systemd[1]: Started Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:27:23.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:23 smithi116 ceph-mon[23713]: mon.smithi116 is new leader, mons smithi116 in quorum (ranks 0) 2024-09-20T19:27:23.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:23 smithi116 ceph-mon[23713]: monmap e1: 1 mons at {smithi116=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0]} 2024-09-20T19:27:23.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:23 smithi116 ceph-mon[23713]: fsmap 2024-09-20T19:27:23.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:23 smithi116 ceph-mon[23713]: osdmap e1: 0 total, 0 up, 0 in 2024-09-20T19:27:23.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:23 smithi116 ceph-mon[23713]: mgrmap e1: no daemons active 2024-09-20T19:27:24.023 INFO:teuthology.orchestra.run.smithi116.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-20T19:27:24.023 INFO:teuthology.orchestra.run.smithi116.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-20T19:27:24.023 INFO:teuthology.orchestra.run.smithi116.stderr:Creating mgr... 2024-09-20T19:27:24.024 INFO:teuthology.orchestra.run.smithi116.stderr:Verifying port 9283 ... 2024-09-20T19:27:24.201 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Failed to reset failed state of unit ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mgr.smithi116.poxcrq.service: Unit ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mgr.smithi116.poxcrq.service not loaded. 2024-09-20T19:27:24.209 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: Created symlink /etc/systemd/system/ceph-2d2015e0-7786-11ef-baf5-efdc52797490.target.wants/ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mgr.smithi116.poxcrq.service → /etc/systemd/system/ceph-2d2015e0-7786-11ef-baf5-efdc52797490@.service. 2024-09-20T19:27:24.790 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:24 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2212482374' entity='client.admin' 2024-09-20T19:27:24.903 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: disabled 2024-09-20T19:27:24.911 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: inactive 2024-09-20T19:27:24.912 INFO:teuthology.orchestra.run.smithi116.stderr:firewalld.service is not enabled 2024-09-20T19:27:24.912 INFO:teuthology.orchestra.run.smithi116.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T19:27:24.920 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: disabled 2024-09-20T19:27:24.927 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: inactive 2024-09-20T19:27:24.928 INFO:teuthology.orchestra.run.smithi116.stderr:firewalld.service is not enabled 2024-09-20T19:27:24.928 INFO:teuthology.orchestra.run.smithi116.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-20T19:27:24.928 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for mgr to start... 2024-09-20T19:27:24.929 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for mgr... 2024-09-20T19:27:25.493 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:25.493 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:27:25.493 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsid": "2d2015e0-7786-11ef-baf5-efdc52797490", 2024-09-20T19:27:25.493 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "health": { 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 0 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "smithi116" 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:25.494 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T19:27:25.495 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "available": false, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "iostat", 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "nfs", 2024-09-20T19:27:25.496 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "restful" 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modified": "2024-09-20T19:27:18.183149+0000", 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T19:27:25.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:27:25.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:25 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3060480222' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T19:27:26.083 INFO:teuthology.orchestra.run.smithi116.stderr:mgr not available, waiting (1/15)... 2024-09-20T19:27:28.570 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsid": "2d2015e0-7786-11ef-baf5-efdc52797490", 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "health": { 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 0 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T19:27:28.571 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "smithi116" 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T19:27:28.572 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T19:27:28.575 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T19:27:28.575 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T19:27:28.575 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:28.575 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "available": false, 2024-09-20T19:27:28.576 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "iostat", 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "nfs", 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "restful" 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modified": "2024-09-20T19:27:18.183149+0000", 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T19:27:28.577 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:27:29.120 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:28 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3010585031' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T19:27:29.256 INFO:teuthology.orchestra.run.smithi116.stderr:mgr not available, waiting (2/15)... 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: Activating manager daemon smithi116.poxcrq 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: mgrmap e2: smithi116.poxcrq(active, starting, since 0.00278031s) 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:27:31.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:31.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:31.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:30 smithi116 ceph-mon[23713]: from='mgr.14100 172.21.15.116:0/3022935559' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:31.726 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:31.726 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsid": "2d2015e0-7786-11ef-baf5-efdc52797490", 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "health": { 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 0 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "smithi116" 2024-09-20T19:27:31.727 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T19:27:31.728 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T19:27:31.731 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T19:27:31.731 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T19:27:31.731 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:31.731 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T19:27:31.732 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "available": false, 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "iostat", 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "nfs", 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "restful" 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modified": "2024-09-20T19:27:18.183149+0000", 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T19:27:31.733 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:27:31.888 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:31 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2496634730' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T19:27:31.889 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:31 smithi116 ceph-mon[23713]: mgrmap e3: smithi116.poxcrq(active, since 1.00597s) 2024-09-20T19:27:32.328 INFO:teuthology.orchestra.run.smithi116.stderr:mgr not available, waiting (3/15)... 2024-09-20T19:27:34.227 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:33 smithi116 ceph-mon[23713]: mgrmap e4: smithi116.poxcrq(active, since 2s) 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsid": "2d2015e0-7786-11ef-baf5-efdc52797490", 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "health": { 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T19:27:34.977 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 0 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "smithi116" 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T19:27:34.978 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T19:27:34.981 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T19:27:34.981 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T19:27:34.981 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:34.982 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "available": true, 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "iostat", 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "nfs", 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "restful" 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ], 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "modified": "2024-09-20T19:27:18.183149+0000", 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "services": {} 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: }, 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T19:27:34.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:27:35.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:35 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/834409052' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T19:27:35.526 INFO:teuthology.orchestra.run.smithi116.stderr:mgr is available 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: [global] 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: fsid = 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T19:27:36.158 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:36.159 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: [mgr] 2024-09-20T19:27:36.159 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T19:27:36.159 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 2024-09-20T19:27:36.159 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: [osd] 2024-09-20T19:27:36.159 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T19:27:36.159 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T19:27:36.462 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:36 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/19717341' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T19:27:36.749 INFO:teuthology.orchestra.run.smithi116.stderr:Enabling cephadm module... 2024-09-20T19:27:37.727 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:37 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1158877271' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-20T19:27:38.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:38 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1158877271' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-20T19:27:38.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:38 smithi116 ceph-mon[23713]: mgrmap e5: smithi116.poxcrq(active, since 6s) 2024-09-20T19:27:38.674 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:27:38.674 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-20T19:27:38.674 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "available": true, 2024-09-20T19:27:38.674 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "active_name": "smithi116.poxcrq", 2024-09-20T19:27:38.674 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T19:27:38.674 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:27:39.244 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for the mgr to restart... 2024-09-20T19:27:39.244 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for mgr epoch 5... 2024-09-20T19:27:39.515 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:39 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1200316804' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T19:27:44.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:43 smithi116 ceph-mon[23713]: Active manager daemon smithi116.poxcrq restarted 2024-09-20T19:27:44.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:43 smithi116 ceph-mon[23713]: Activating manager daemon smithi116.poxcrq 2024-09-20T19:27:44.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:43 smithi116 ceph-mon[23713]: osdmap e2: 0 total, 0 up, 0 in 2024-09-20T19:27:44.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:27:44.983 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-20T19:27:44.984 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T19:27:44.984 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: mgrmap e6: smithi116.poxcrq(active, starting, since 0.515619s) 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:27:45.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:27:45.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:27:45.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:27:45.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:27:45.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:45.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:44 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:27:45.560 INFO:teuthology.orchestra.run.smithi116.stderr:mgr epoch 5 is available 2024-09-20T19:27:45.560 INFO:teuthology.orchestra.run.smithi116.stderr:Setting orchestrator backend to cephadm... 2024-09-20T19:27:46.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:45 smithi116 ceph-mon[23713]: [20/Sep/2024:19:27:44] ENGINE Bus STARTING 2024-09-20T19:27:46.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:45 smithi116 ceph-mon[23713]: [20/Sep/2024:19:27:44] ENGINE Serving on https://172.21.15.116:7150 2024-09-20T19:27:46.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:45 smithi116 ceph-mon[23713]: [20/Sep/2024:19:27:44] ENGINE Bus STARTED 2024-09-20T19:27:46.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:45 smithi116 ceph-mon[23713]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-20T19:27:46.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:45 smithi116 ceph-mon[23713]: mgrmap e7: smithi116.poxcrq(active, since 1.51997s) 2024-09-20T19:27:46.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:45 smithi116 ceph-mon[23713]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-20T19:27:47.214 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:46 smithi116 ceph-mon[23713]: mgrmap e8: smithi116.poxcrq(active, since 2s) 2024-09-20T19:27:47.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:46 smithi116 ceph-mon[23713]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:47.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:46 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:47.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:46 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:27:47.403 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: value unchanged 2024-09-20T19:27:47.904 INFO:teuthology.orchestra.run.smithi116.stderr:Generating ssh key... 2024-09-20T19:27:48.203 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:47 smithi116 ceph-mon[23713]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:50.009 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxug61JjuyjE4wIWVTWKg8G7YQj0urOJ1lQWF0ausONOQ593kdFfHzqqBdVAq6rU6JSaU37JTDrul5Tzrk06HFC+Uvc5FEVNx+VM5fvC8g9X2lF5TY8lvhGhAo9i00FtKVHMYgrbywQ1VtiYz4oldo4BcbJh07F/VtuuBgjV1kPvh2ujMubKwZ/h36hbVg9qolOo680cGnwcvwymum0OHiXgPK1+qJamBS5IwFqhaiaeRSf5nW8xpzIWoUfnjltKTp0A7jc0xB4kX7bU+bqfuBVIUp3a72dYwto+1O3e+yy3SpGn68eWxOknnD9voDdcjVtbCCCe492H4cHpM6EiRaY4pPOLe+PtVZjJxx7X9jMd2tlVlQtVNEKr0giKSKB/qz1JkeLAYjwETlLDD09Rc1wXIhZn49K6HClzqwf+K2IR2HUHO6yxhWbdiZElLfdBMUK/Al6iBFyjPwQwU76BHyylu8YMgEAogB1n2SsQN3HJVtC+3l/40I7MtQJ0N8/2k= ceph-2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:27:50.120 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:49 smithi116 ceph-mon[23713]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:50.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:49 smithi116 ceph-mon[23713]: Generating ssh key... 2024-09-20T19:27:50.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:49 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:50.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:49 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:50.632 INFO:teuthology.orchestra.run.smithi116.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-20T19:27:50.632 INFO:teuthology.orchestra.run.smithi116.stderr:Adding key to root@localhost authorized_keys... 2024-09-20T19:27:50.636 INFO:teuthology.orchestra.run.smithi116.stderr:Adding host smithi116... 2024-09-20T19:27:50.853 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:50 smithi116 ceph-mon[23713]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:52.057 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:51 smithi116 ceph-mon[23713]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi116", "addr": "172.21.15.116", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:52.068 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Added host 'smithi116' with addr '172.21.15.116' 2024-09-20T19:27:52.615 INFO:teuthology.orchestra.run.smithi116.stderr:Deploying mon service with default placement... 2024-09-20T19:27:53.120 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:53 smithi116 ceph-mon[23713]: Deploying cephadm binary to smithi116 2024-09-20T19:27:53.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:53 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:53.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:53 smithi116 ceph-mon[23713]: Added host smithi116 2024-09-20T19:27:53.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:53 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:27:53.323 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-20T19:27:54.083 INFO:teuthology.orchestra.run.smithi116.stderr:Deploying mgr service with default placement... 2024-09-20T19:27:54.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:54 smithi116 ceph-mon[23713]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:54.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:54 smithi116 ceph-mon[23713]: Saving service mon spec with placement count:5 2024-09-20T19:27:54.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:54 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:54.717 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-20T19:27:55.458 INFO:teuthology.orchestra.run.smithi116.stderr:Deploying crash service with default placement... 2024-09-20T19:27:55.801 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:55 smithi116 ceph-mon[23713]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:55.801 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:55 smithi116 ceph-mon[23713]: Saving service mgr spec with placement count:2 2024-09-20T19:27:55.801 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:55 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:56.098 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-20T19:27:56.976 INFO:teuthology.orchestra.run.smithi116.stderr:Deploying prometheus service with default placement... 2024-09-20T19:27:57.270 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:57 smithi116 ceph-mon[23713]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:57.270 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:57 smithi116 ceph-mon[23713]: Saving service crash spec with placement * 2024-09-20T19:27:57.271 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:57 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:57.271 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:57 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:57.271 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:57 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:57.574 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-20T19:27:58.844 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:58 smithi116 ceph-mon[23713]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:27:58.844 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:58 smithi116 ceph-mon[23713]: Saving service prometheus spec with placement count:1 2024-09-20T19:27:58.844 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:58 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:58.844 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:58 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:27:58.846 INFO:teuthology.orchestra.run.smithi116.stderr:Deploying grafana service with default placement... 2024-09-20T19:27:59.551 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-20T19:28:00.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:59 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:00.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:27:59 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:00.632 INFO:teuthology.orchestra.run.smithi116.stderr:Deploying node-exporter service with default placement... 2024-09-20T19:28:01.209 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:00 smithi116 ceph-mon[23713]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:01.210 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:00 smithi116 ceph-mon[23713]: Saving service grafana spec with placement count:1 2024-09-20T19:28:01.508 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-20T19:28:02.182 INFO:teuthology.orchestra.run.smithi116.stderr:Deploying alertmanager service with default placement... 2024-09-20T19:28:02.828 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:02 smithi116 ceph-mon[23713]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:02.828 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:02 smithi116 ceph-mon[23713]: Saving service node-exporter spec with placement * 2024-09-20T19:28:02.828 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:02 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:02.829 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-20T19:28:04.053 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:03 smithi116 ceph-mon[23713]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:04.053 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:03 smithi116 ceph-mon[23713]: Saving service alertmanager spec with placement count:1 2024-09-20T19:28:04.053 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:03 smithi116 ceph-mon[23713]: from='mgr.14120 172.21.15.116:0/1703701577' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:05.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:05 smithi116 ceph-mon[23713]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:05.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:05 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3824615961' entity='client.admin' 2024-09-20T19:28:05.825 INFO:teuthology.orchestra.run.smithi116.stderr:Enabling the dashboard module... 2024-09-20T19:28:06.620 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:06 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3762616770' entity='client.admin' 2024-09-20T19:28:07.471 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:07 smithi116 ceph-mon[23713]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:07.472 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:07 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/658336115' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-20T19:28:08.323 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:08 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/658336115' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-20T19:28:08.324 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:08 smithi116 ceph-mon[23713]: mgrmap e9: smithi116.poxcrq(active, since 23s) 2024-09-20T19:28:08.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:28:08.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-20T19:28:08.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "available": true, 2024-09-20T19:28:08.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "active_name": "smithi116.poxcrq", 2024-09-20T19:28:08.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T19:28:08.497 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:28:09.147 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for the mgr to restart... 2024-09-20T19:28:09.147 INFO:teuthology.orchestra.run.smithi116.stderr:Waiting for mgr epoch 9... 2024-09-20T19:28:09.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:09 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/350181370' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T19:28:14.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:13 smithi116 ceph-mon[23713]: Active manager daemon smithi116.poxcrq restarted 2024-09-20T19:28:14.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:13 smithi116 ceph-mon[23713]: Activating manager daemon smithi116.poxcrq 2024-09-20T19:28:14.268 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:13 smithi116 ceph-mon[23713]: osdmap e3: 0 total, 0 up, 0 in 2024-09-20T19:28:14.988 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: { 2024-09-20T19:28:14.988 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-20T19:28:14.989 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T19:28:14.989 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: } 2024-09-20T19:28:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: mgrmap e10: smithi116.poxcrq(active, starting, since 0.768463s) 2024-09-20T19:28:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:28:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:28:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:28:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:28:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:28:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:28:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:28:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:28:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:28:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:28:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:14 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:15.707 INFO:teuthology.orchestra.run.smithi116.stderr:mgr epoch 9 is available 2024-09-20T19:28:15.707 INFO:teuthology.orchestra.run.smithi116.stderr:Generating a dashboard self-signed certificate... 2024-09-20T19:28:16.238 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:15 smithi116 ceph-mon[23713]: [20/Sep/2024:19:28:14] ENGINE Bus STARTING 2024-09-20T19:28:16.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:15 smithi116 ceph-mon[23713]: [20/Sep/2024:19:28:14] ENGINE Serving on https://172.21.15.116:7150 2024-09-20T19:28:16.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:15 smithi116 ceph-mon[23713]: [20/Sep/2024:19:28:14] ENGINE Bus STARTED 2024-09-20T19:28:16.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:15 smithi116 ceph-mon[23713]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-20T19:28:16.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:15 smithi116 ceph-mon[23713]: mgrmap e11: smithi116.poxcrq(active, since 1.77091s) 2024-09-20T19:28:16.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:15 smithi116 ceph-mon[23713]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-20T19:28:16.466 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-20T19:28:17.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:16 smithi116 ceph-mon[23713]: mgrmap e12: smithi116.poxcrq(active, since 2s) 2024-09-20T19:28:17.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:16 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:17.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:16 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:17.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:16 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:17.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:16 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:17.130 INFO:teuthology.orchestra.run.smithi116.stderr:Creating initial admin user... 2024-09-20T19:28:18.081 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$P0oj9pNhrJVs6yHjd7fwYe8Rl5iC9BS.Ts53XdEVyXM/brODAXQ7O", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726860498, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-20T19:28:18.095 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:17 smithi116 ceph-mon[23713]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:19.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:19 smithi116 ceph-mon[23713]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:19.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:19 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:19.418 INFO:teuthology.orchestra.run.smithi116.stderr:Fetching dashboard port number... 2024-09-20T19:28:20.063 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: 8443 2024-09-20T19:28:20.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:20 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/926424000' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-20T19:28:20.718 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: disabled 2024-09-20T19:28:20.725 INFO:teuthology.orchestra.run.smithi116.stderr:systemctl: inactive 2024-09-20T19:28:20.726 INFO:teuthology.orchestra.run.smithi116.stderr:firewalld.service is not enabled 2024-09-20T19:28:20.726 INFO:teuthology.orchestra.run.smithi116.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-20T19:28:20.728 INFO:teuthology.orchestra.run.smithi116.stderr:Ceph Dashboard is now available at: 2024-09-20T19:28:20.728 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:20.728 INFO:teuthology.orchestra.run.smithi116.stderr: URL: https://smithi116.front.sepia.ceph.com:8443/ 2024-09-20T19:28:20.728 INFO:teuthology.orchestra.run.smithi116.stderr: User: admin 2024-09-20T19:28:20.728 INFO:teuthology.orchestra.run.smithi116.stderr: Password: mn5g53a4oh 2024-09-20T19:28:20.728 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:20.728 INFO:teuthology.orchestra.run.smithi116.stderr:Enabling autotune for osd_memory_target 2024-09-20T19:28:21.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:21 smithi116 ceph-mon[23713]: mgrmap e13: smithi116.poxcrq(active, since 6s) 2024-09-20T19:28:22.753 INFO:teuthology.orchestra.run.smithi116.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr:You can access the Ceph CLI with: 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr: ceph telemetry on 2024-09-20T19:28:23.423 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:23.424 INFO:teuthology.orchestra.run.smithi116.stderr:For more information see: 2024-09-20T19:28:23.424 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:23.424 INFO:teuthology.orchestra.run.smithi116.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-20T19:28:23.424 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:28:23.424 INFO:teuthology.orchestra.run.smithi116.stderr:Bootstrap complete. 2024-09-20T19:28:23.463 INFO:tasks.cephadm:Fetching config... 2024-09-20T19:28:23.463 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:28:23.463 DEBUG:teuthology.orchestra.run.smithi116:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-20T19:28:23.479 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-20T19:28:23.479 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:28:23.480 DEBUG:teuthology.orchestra.run.smithi116:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-20T19:28:23.535 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-20T19:28:23.535 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:28:23.535 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/keyring of=/dev/stdout 2024-09-20T19:28:23.601 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-20T19:28:23.601 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:28:23.601 DEBUG:teuthology.orchestra.run.smithi116:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-20T19:28:23.658 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-20T19:28:23.658 DEBUG:teuthology.orchestra.run.smithi116:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxug61JjuyjE4wIWVTWKg8G7YQj0urOJ1lQWF0ausONOQ593kdFfHzqqBdVAq6rU6JSaU37JTDrul5Tzrk06HFC+Uvc5FEVNx+VM5fvC8g9X2lF5TY8lvhGhAo9i00FtKVHMYgrbywQ1VtiYz4oldo4BcbJh07F/VtuuBgjV1kPvh2ujMubKwZ/h36hbVg9qolOo680cGnwcvwymum0OHiXgPK1+qJamBS5IwFqhaiaeRSf5nW8xpzIWoUfnjltKTp0A7jc0xB4kX7bU+bqfuBVIUp3a72dYwto+1O3e+yy3SpGn68eWxOknnD9voDdcjVtbCCCe492H4cHpM6EiRaY4pPOLe+PtVZjJxx7X9jMd2tlVlQtVNEKr0giKSKB/qz1JkeLAYjwETlLDD09Rc1wXIhZn49K6HClzqwf+K2IR2HUHO6yxhWbdiZElLfdBMUK/Al6iBFyjPwQwU76BHyylu8YMgEAogB1n2SsQN3HJVtC+3l/40I7MtQJ0N8/2k= ceph-2d2015e0-7786-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T19:28:23.733 INFO:teuthology.orchestra.run.smithi116.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxug61JjuyjE4wIWVTWKg8G7YQj0urOJ1lQWF0ausONOQ593kdFfHzqqBdVAq6rU6JSaU37JTDrul5Tzrk06HFC+Uvc5FEVNx+VM5fvC8g9X2lF5TY8lvhGhAo9i00FtKVHMYgrbywQ1VtiYz4oldo4BcbJh07F/VtuuBgjV1kPvh2ujMubKwZ/h36hbVg9qolOo680cGnwcvwymum0OHiXgPK1+qJamBS5IwFqhaiaeRSf5nW8xpzIWoUfnjltKTp0A7jc0xB4kX7bU+bqfuBVIUp3a72dYwto+1O3e+yy3SpGn68eWxOknnD9voDdcjVtbCCCe492H4cHpM6EiRaY4pPOLe+PtVZjJxx7X9jMd2tlVlQtVNEKr0giKSKB/qz1JkeLAYjwETlLDD09Rc1wXIhZn49K6HClzqwf+K2IR2HUHO6yxhWbdiZElLfdBMUK/Al6iBFyjPwQwU76BHyylu8YMgEAogB1n2SsQN3HJVtC+3l/40I7MtQJ0N8/2k= ceph-2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:28:23.744 DEBUG:teuthology.orchestra.run.smithi118:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxug61JjuyjE4wIWVTWKg8G7YQj0urOJ1lQWF0ausONOQ593kdFfHzqqBdVAq6rU6JSaU37JTDrul5Tzrk06HFC+Uvc5FEVNx+VM5fvC8g9X2lF5TY8lvhGhAo9i00FtKVHMYgrbywQ1VtiYz4oldo4BcbJh07F/VtuuBgjV1kPvh2ujMubKwZ/h36hbVg9qolOo680cGnwcvwymum0OHiXgPK1+qJamBS5IwFqhaiaeRSf5nW8xpzIWoUfnjltKTp0A7jc0xB4kX7bU+bqfuBVIUp3a72dYwto+1O3e+yy3SpGn68eWxOknnD9voDdcjVtbCCCe492H4cHpM6EiRaY4pPOLe+PtVZjJxx7X9jMd2tlVlQtVNEKr0giKSKB/qz1JkeLAYjwETlLDD09Rc1wXIhZn49K6HClzqwf+K2IR2HUHO6yxhWbdiZElLfdBMUK/Al6iBFyjPwQwU76BHyylu8YMgEAogB1n2SsQN3HJVtC+3l/40I7MtQJ0N8/2k= ceph-2d2015e0-7786-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T19:28:23.871 INFO:teuthology.orchestra.run.smithi118.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxug61JjuyjE4wIWVTWKg8G7YQj0urOJ1lQWF0ausONOQ593kdFfHzqqBdVAq6rU6JSaU37JTDrul5Tzrk06HFC+Uvc5FEVNx+VM5fvC8g9X2lF5TY8lvhGhAo9i00FtKVHMYgrbywQ1VtiYz4oldo4BcbJh07F/VtuuBgjV1kPvh2ujMubKwZ/h36hbVg9qolOo680cGnwcvwymum0OHiXgPK1+qJamBS5IwFqhaiaeRSf5nW8xpzIWoUfnjltKTp0A7jc0xB4kX7bU+bqfuBVIUp3a72dYwto+1O3e+yy3SpGn68eWxOknnD9voDdcjVtbCCCe492H4cHpM6EiRaY4pPOLe+PtVZjJxx7X9jMd2tlVlQtVNEKr0giKSKB/qz1JkeLAYjwETlLDD09Rc1wXIhZn49K6HClzqwf+K2IR2HUHO6yxhWbdiZElLfdBMUK/Al6iBFyjPwQwU76BHyylu8YMgEAogB1n2SsQN3HJVtC+3l/40I7MtQJ0N8/2k= ceph-2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:28:23.885 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-20T19:28:24.120 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:23 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/906030286' entity='client.admin' 2024-09-20T19:28:27.133 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-20T19:28:27.133 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-20T19:28:27.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:27 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2193293906' entity='client.admin' 2024-09-20T19:28:29.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:29 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:29.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:29 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd/host:smithi116", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:28:29.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:29 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:29.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:29 smithi116 ceph-mon[23713]: Deploying daemon alertmanager.smithi116 on smithi116 2024-09-20T19:28:30.301 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi118 2024-09-20T19:28:30.302 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-20T19:28:30.302 DEBUG:teuthology.orchestra.run.smithi118:> dd of=/etc/ceph/ceph.conf 2024-09-20T19:28:30.321 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-20T19:28:30.321 DEBUG:teuthology.orchestra.run.smithi118:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:28:30.380 INFO:tasks.cephadm:Adding host smithi118 to orchestrator... 2024-09-20T19:28:30.380 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch host add smithi118 2024-09-20T19:28:30.870 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:30 smithi116 ceph-mon[23713]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:30.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:30 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:34.412 INFO:teuthology.orchestra.run.smithi116.stdout:Added host 'smithi118' with addr '172.21.15.118' 2024-09-20T19:28:35.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:35 smithi116 ceph-mon[23713]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi118", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:35.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:35 smithi116 ceph-mon[23713]: Deploying cephadm binary to smithi118 2024-09-20T19:28:35.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:35 smithi116 ceph-mon[23713]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:35.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:35 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:36.077 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch host ls --format=json 2024-09-20T19:28:36.334 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:36 smithi116 ceph-mon[23713]: Added host smithi118 2024-09-20T19:28:37.359 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:37 smithi116 ceph-mon[23713]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:39.089 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:28:39.090 INFO:teuthology.orchestra.run.smithi116.stdout:[{"addr": "172.21.15.116", "hostname": "smithi116", "labels": [], "status": ""}, {"addr": "172.21.15.118", "hostname": "smithi118", "labels": [], "status": ""}] 2024-09-20T19:28:39.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:38 smithi116 ceph-mon[23713]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:39.874 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-20T19:28:39.874 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd crush tunables default 2024-09-20T19:28:40.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:39 smithi116 ceph-mon[23713]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:28:41.347 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:41 smithi116 ceph-mon[23713]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:41.348 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:41 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:41.348 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:41 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T19:28:41.348 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:41 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-20T19:28:41.348 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:41 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:28:41.348 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:41 smithi116 ceph-mon[23713]: Deploying daemon crash.smithi116 on smithi116 2024-09-20T19:28:43.163 INFO:teuthology.orchestra.run.smithi116.stderr:adjusted tunables profile to default 2024-09-20T19:28:43.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:43 smithi116 ceph-mon[23713]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:43.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:43 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1727826410' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-20T19:28:43.828 INFO:tasks.cephadm:Adding mon.smithi116 on smithi116 2024-09-20T19:28:43.828 INFO:tasks.cephadm:Adding mon.smithi118 on smithi118 2024-09-20T19:28:43.828 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch apply mon '2;smithi116:172.21.15.116=smithi116;smithi118:172.21.15.118=smithi118' 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1727826410' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:44.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:44 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:45.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:45 smithi116 ceph-mon[23713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T19:28:45.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:45 smithi116 ceph-mon[23713]: Deploying daemon grafana.smithi116 on smithi116 2024-09-20T19:28:45.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:45 smithi116 ceph-mon[23713]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:45.781 INFO:teuthology.orchestra.run.smithi118.stdout:Scheduled mon update... 2024-09-20T19:28:46.351 DEBUG:teuthology.orchestra.run.smithi118:mon.smithi118> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi118.service 2024-09-20T19:28:46.353 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:28:46.353 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:28:47.120 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:46 smithi116 ceph-mon[23713]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi116:172.21.15.116=smithi116;smithi118:172.21.15.118=smithi118", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:28:47.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:46 smithi116 ceph-mon[23713]: Saving service mon spec with placement smithi116:172.21.15.116=smithi116;smithi118:172.21.15.118=smithi118;count:2 2024-09-20T19:28:47.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:46 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:47.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:46 smithi116 ceph-mon[23713]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:48.131 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:28:48.163 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:28:48.163 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:28:49.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:48 smithi116 ceph-mon[23713]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:49.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:48 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/3046263837' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:28:49.657 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:28:49.657 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:28:51.192 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:50 smithi116 ceph-mon[23713]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:51.650 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:28:51.650 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:28:51.652 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:28:52.098 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:51 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2837400900' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:28:53.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:52 smithi116 ceph-mon[23713]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:53.234 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:28:53.234 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:28:55.120 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:28:55.120 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:28:55.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:54 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:55.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:54 smithi116 ceph-mon[23713]: Deploying daemon node-exporter.smithi116 on smithi116 2024-09-20T19:28:55.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:54 smithi116 ceph-mon[23713]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:55.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:54 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:55.122 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:28:56.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:55 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/199300897' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:28:56.706 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:28:56.707 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:28:57.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:56 smithi116 ceph-mon[23713]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:28:57.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:56 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' 2024-09-20T19:28:57.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:56 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-20T19:28:58.609 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:28:58.609 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:28:58.611 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:28:58.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:58 smithi116 ceph-mon[23713]: from='mgr.14162 172.21.15.116:0/1820235659' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-20T19:28:58.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:58 smithi116 ceph-mon[23713]: mgrmap e14: smithi116.poxcrq(active, since 44s) 2024-09-20T19:28:59.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:28:59 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/674890106' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:00.294 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:29:00.295 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:29:02.106 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:29:02.106 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:29:02.108 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:29:02.290 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:02 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/420873505' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:03.689 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:29:03.690 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:29:04.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: Active manager daemon smithi116.poxcrq restarted 2024-09-20T19:29:04.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: Activating manager daemon smithi116.poxcrq 2024-09-20T19:29:04.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: osdmap e5: 0 total, 0 up, 0 in 2024-09-20T19:29:04.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: mgrmap e15: smithi116.poxcrq(active, starting, since 0.426692s) 2024-09-20T19:29:04.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:29:04.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:29:04.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:29:04.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:05.629 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:29:05.629 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:29:05.631 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:29:06.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:05 smithi116 ceph-mon[23713]: [20/Sep/2024:19:29:04] ENGINE Bus STARTING 2024-09-20T19:29:06.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:05 smithi116 ceph-mon[23713]: [20/Sep/2024:19:29:04] ENGINE Serving on https://172.21.15.116:7150 2024-09-20T19:29:06.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:05 smithi116 ceph-mon[23713]: [20/Sep/2024:19:29:04] ENGINE Bus STARTED 2024-09-20T19:29:06.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:06.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:05 smithi116 ceph-mon[23713]: mgrmap e16: smithi116.poxcrq(active, since 1.42824s) 2024-09-20T19:29:06.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:06.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:05 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2536856509' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:07.230 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:29:07.230 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:29:07.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:07.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:07 smithi116 ceph-mon[23713]: mgrmap e17: smithi116.poxcrq(active, since 2s) 2024-09-20T19:29:10.332 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:29:10.332 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:29:10.334 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:29:10.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:10.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:10.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:10.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd/host:smithi118", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd/host:smithi116", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2665832265' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:10.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:10 smithi116 ceph-mon[23713]: Deploying daemon crash.smithi118 on smithi118 2024-09-20T19:29:11.931 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:29:11.932 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:29:14.434 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:29:14.435 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:29:14.436 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:29:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:29:15.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-20T19:29:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:29:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: Deploying daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:29:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:14 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/4074143813' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:16.032 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:29:16.032 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:29:18.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:18.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T19:29:18.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:18.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:17 smithi116 ceph-mon[23713]: Deploying daemon mon.smithi118 on smithi118 2024-09-20T19:29:19.131 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:29:19.131 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":1,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:27:15.298011Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T19:29:19.134 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 1 2024-09-20T19:29:20.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:20 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:20.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:20 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/518967761' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:20.728 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T19:29:20.728 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T19:29:23.043 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:23 smithi118 ceph-mon[25856]: mon.smithi118@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-20T19:29:23.043 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:23 smithi118 bash[25760]: 849316daa404d4446dce5e8d6973a1e0c81afe0ebe76b4caf8f8d87e9c59d2ab 2024-09-20T19:29:23.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:22 smithi116 ceph-mon[23713]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:29:23.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:22 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.118:0/2150095817' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rmiazv/crt"}]: dispatch 2024-09-20T19:29:23.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:22 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.118:0/2150095817' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T19:29:23.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:22 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.118:0/2150095817' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi118.rmiazv/key"}]: dispatch 2024-09-20T19:29:23.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:22 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.118:0/2150095817' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T19:29:23.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:23 smithi118 systemd[1]: Started Ceph mon.smithi118 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:29:28.033 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:29:28.033 INFO:teuthology.orchestra.run.smithi118.stdout:{"epoch":2,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","modified":"2024-09-20T19:29:23.022718Z","created":"2024-09-20T19:27:15.298011Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi116","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:3300","nonce":0},{"type":"v1","addr":"172.21.15.116:6789","nonce":0}]},"addr":"172.21.15.116:6789/0","public_addr":"172.21.15.116:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi118","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:3300","nonce":0},{"type":"v1","addr":"172.21.15.118:6789","nonce":0}]},"addr":"172.21.15.118:6789/0","public_addr":"172.21.15.118:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-20T19:29:28.036 INFO:teuthology.orchestra.run.smithi118.stderr:dumped monmap epoch 2 2024-09-20T19:29:28.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:29:28.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: mon.smithi116 calling monitor election 2024-09-20T19:29:28.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/1831942035' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:28.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: mon.smithi118 calling monitor election 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: mon.smithi116 is new leader, mons smithi116,smithi118 in quorum (ranks 0,1) 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: monmap e2: 2 mons at {smithi116=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0],smithi118=[v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0]} 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: fsmap 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: osdmap e5: 0 total, 0 up, 0 in 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: mgrmap e18: smithi116.poxcrq(active, since 24s), standbys: smithi118.rmiazv 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: overall HEALTH_OK 2024-09-20T19:29:28.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:28 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: mon.smithi116 calling monitor election 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/1831942035' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: mon.smithi118 calling monitor election 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.397 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: mon.smithi116 is new leader, mons smithi116,smithi118 in quorum (ranks 0,1) 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: monmap e2: 2 mons at {smithi116=[v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0],smithi118=[v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0]} 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: fsmap 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: osdmap e5: 0 total, 0 up, 0 in 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: mgrmap e18: smithi116.poxcrq(active, since 24s), standbys: smithi118.rmiazv 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: overall HEALTH_OK 2024-09-20T19:29:28.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:28 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:28.685 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-20T19:29:28.685 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph config generate-minimal-conf 2024-09-20T19:29:29.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:29 smithi118 ceph-mon[25856]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:29.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:29 smithi118 ceph-mon[25856]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-20T19:29:29.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:29.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:29.304 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:29 smithi116 ceph-mon[23713]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:29.304 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:29 smithi116 ceph-mon[23713]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-20T19:29:29.304 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:29:29.304 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:30.578 INFO:teuthology.orchestra.run.smithi116.stdout:# minimal ceph.conf for 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:29:30.578 INFO:teuthology.orchestra.run.smithi116.stdout:[global] 2024-09-20T19:29:30.578 INFO:teuthology.orchestra.run.smithi116.stdout: fsid = 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:29:30.578 INFO:teuthology.orchestra.run.smithi116.stdout: mon_host = [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] 2024-09-20T19:29:31.203 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-20T19:29:31.204 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:29:31.204 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T19:29:31.232 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:29:31.232 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:29:31.293 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:31 smithi116 ceph-mon[23713]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:31.293 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:31 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1745153697' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:31.297 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-20T19:29:31.297 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T19:29:31.321 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:31 smithi118 ceph-mon[25856]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:31.322 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:31 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1745153697' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:31.326 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-20T19:29:31.326 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:29:31.392 DEBUG:tasks.cephadm:set 0 configs 2024-09-20T19:29:31.393 INFO:tasks.cephadm:Deploying OSDs... 2024-09-20T19:29:31.393 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:29:31.393 DEBUG:teuthology.orchestra.run.smithi116:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T19:29:31.408 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T19:29:31.409 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_1 2024-09-20T19:29:31.463 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T19:29:31.463 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:31.463 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-20T19:29:31.463 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:31.464 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:31.464 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-09-20 19:28:54.400666550 +0000 2024-09-20T19:29:31.464 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-09-20 19:25:42.188997128 +0000 2024-09-20T19:29:31.464 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-09-20 19:25:42.188997128 +0000 2024-09-20T19:29:31.464 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-09-20 19:25:42.188997128 +0000 2024-09-20T19:29:31.464 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T19:29:31.526 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-09-20T19:29:31.527 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-09-20T19:29:31.527 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000325191 s, 1.6 MB/s 2024-09-20T19:29:31.528 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T19:29:31.585 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_2 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-09-20 19:28:54.401666534 +0000 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-09-20 19:25:42.512991651 +0000 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-09-20 19:25:42.512991651 +0000 2024-09-20T19:29:31.640 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-09-20 19:25:42.512991651 +0000 2024-09-20T19:29:31.640 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T19:29:31.702 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-09-20T19:29:31.702 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-09-20T19:29:31.702 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000209302 s, 2.4 MB/s 2024-09-20T19:29:31.703 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T19:29:31.759 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_3 2024-09-20T19:29:31.814 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-09-20 19:28:54.401666534 +0000 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-09-20 19:25:42.818986479 +0000 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-09-20 19:25:42.818986479 +0000 2024-09-20T19:29:31.815 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-09-20 19:25:42.818986479 +0000 2024-09-20T19:29:31.815 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T19:29:31.877 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-09-20T19:29:31.877 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-09-20T19:29:31.877 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000177 s, 2.9 MB/s 2024-09-20T19:29:31.879 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T19:29:31.934 DEBUG:teuthology.orchestra.run.smithi116:> stat /dev/vg_nvme/lv_4 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout:Access: 2024-09-20 19:28:54.401666534 +0000 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout:Modify: 2024-09-20 19:25:43.130981205 +0000 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout:Change: 2024-09-20 19:25:43.130981205 +0000 2024-09-20T19:29:31.990 INFO:teuthology.orchestra.run.smithi116.stdout: Birth: 2024-09-20 19:25:43.130981205 +0000 2024-09-20T19:29:31.991 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T19:29:32.053 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records in 2024-09-20T19:29:32.053 INFO:teuthology.orchestra.run.smithi116.stderr:1+0 records out 2024-09-20T19:29:32.053 INFO:teuthology.orchestra.run.smithi116.stderr:512 bytes copied, 0.000250408 s, 2.0 MB/s 2024-09-20T19:29:32.054 DEBUG:teuthology.orchestra.run.smithi116:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T19:29:32.110 DEBUG:teuthology.orchestra.run.smithi118:> set -ex 2024-09-20T19:29:32.110 DEBUG:teuthology.orchestra.run.smithi118:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T19:29:32.127 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T19:29:32.127 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_1 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-20 19:29:29.051701555 +0000 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-20 19:25:42.514809736 +0000 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-20 19:25:42.514809736 +0000 2024-09-20T19:29:32.184 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-20 19:25:42.514809736 +0000 2024-09-20T19:29:32.185 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T19:29:32.247 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-20T19:29:32.248 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-20T19:29:32.248 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 9.8518e-05 s, 5.2 MB/s 2024-09-20T19:29:32.249 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T19:29:32.305 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_2 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-20 19:29:29.051701555 +0000 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-20 19:25:42.836802107 +0000 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-20 19:25:42.836802107 +0000 2024-09-20T19:29:32.361 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-20 19:25:42.836802107 +0000 2024-09-20T19:29:32.361 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T19:29:32.423 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-20T19:29:32.423 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-20T19:29:32.424 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 9.2177e-05 s, 5.6 MB/s 2024-09-20T19:29:32.425 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T19:29:32.480 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_3 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-20 19:29:29.051701555 +0000 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-20 19:25:43.147794738 +0000 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-20 19:25:43.147794738 +0000 2024-09-20T19:29:32.536 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-20 19:25:43.147794738 +0000 2024-09-20T19:29:32.537 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T19:29:32.598 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-20T19:29:32.599 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-20T19:29:32.599 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 0.000113282 s, 4.5 MB/s 2024-09-20T19:29:32.600 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T19:29:32.655 DEBUG:teuthology.orchestra.run.smithi118:> stat /dev/vg_nvme/lv_4 2024-09-20T19:29:32.710 INFO:teuthology.orchestra.run.smithi118.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout:Access: 2024-09-20 19:29:29.052701532 +0000 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout:Modify: 2024-09-20 19:25:43.461787298 +0000 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout:Change: 2024-09-20 19:25:43.461787298 +0000 2024-09-20T19:29:32.711 INFO:teuthology.orchestra.run.smithi118.stdout: Birth: 2024-09-20 19:25:43.461787298 +0000 2024-09-20T19:29:32.711 DEBUG:teuthology.orchestra.run.smithi118:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T19:29:32.774 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records in 2024-09-20T19:29:32.774 INFO:teuthology.orchestra.run.smithi118.stderr:1+0 records out 2024-09-20T19:29:32.774 INFO:teuthology.orchestra.run.smithi118.stderr:512 bytes copied, 9.079e-05 s, 5.6 MB/s 2024-09-20T19:29:32.775 DEBUG:teuthology.orchestra.run.smithi118:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T19:29:32.830 INFO:tasks.cephadm:Deploying osd.0 on smithi116 with /dev/vg_nvme/lv_4... 2024-09-20T19:29:32.831 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T19:29:33.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:33.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:32 smithi116 ceph-mon[23713]: Deploying daemon prometheus.smithi116 on smithi116 2024-09-20T19:29:33.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:32 smithi116 ceph-mon[23713]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:33.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:33.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:32 smithi118 ceph-mon[25856]: Deploying daemon prometheus.smithi116 on smithi116 2024-09-20T19:29:33.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:32 smithi118 ceph-mon[25856]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:35.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:35 smithi116 ceph-mon[23713]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:35.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:35 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:35.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:35 smithi118 ceph-mon[25856]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:35.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:35 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:37.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:37 smithi116 ceph-mon[23713]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:37.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:37 smithi118 ceph-mon[25856]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:39.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:38 smithi116 ceph-mon[23713]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:39.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:38 smithi118 ceph-mon[25856]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:41.276 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:41 smithi116 ceph-mon[23713]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:41.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:41 smithi118 ceph-mon[25856]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:43.289 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:43 smithi116 ceph-mon[23713]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:43.289 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:43.290 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:29:43.290 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:43.290 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:29:43.408 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:43 smithi118 ceph-mon[25856]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:43.408 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:43.408 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:29:43.408 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:43.408 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:29:43.568 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:29:43.588 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi116:vg_nvme/lv_4 2024-09-20T19:29:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:44 smithi116 ceph-mon[23713]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:29:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:44 smithi116 ceph-mon[23713]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:44 smithi116 ceph-mon[23713]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:29:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:45.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:45.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:45.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:44 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:29:45.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:44 smithi118 ceph-mon[25856]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:45.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:44 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:29:45.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:45.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:47.351 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:47 smithi116 ceph-mon[23713]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:47.352 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:47 smithi116 ceph-mon[23713]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:29:47.352 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:47 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:29:47.352 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:47 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:29:47.352 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:47 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:47 smithi118 ceph-mon[25856]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:47 smithi118 ceph-mon[25856]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:29:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:47 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:29:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:47 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:29:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:47 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T19:29:49.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:49.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:49.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:49.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:29:49.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:49.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:49.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T19:29:49.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T19:29:49.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:50.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:50 smithi118 ceph-mon[25856]: Reconfiguring mon.smithi116 (unknown last config time)... 2024-09-20T19:29:50.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:50 smithi118 ceph-mon[25856]: Reconfiguring daemon mon.smithi116 on smithi116 2024-09-20T19:29:50.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:50 smithi116 ceph-mon[23713]: Reconfiguring mon.smithi116 (unknown last config time)... 2024-09-20T19:29:50.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:50 smithi116 ceph-mon[23713]: Reconfiguring daemon mon.smithi116 on smithi116 2024-09-20T19:29:51.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:51 smithi118 ceph-mon[25856]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:51.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:51 smithi116 ceph-mon[23713]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:52.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:52.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:52 smithi116 ceph-mon[23713]: Reconfiguring mgr.smithi116.poxcrq (unknown last config time)... 2024-09-20T19:29:52.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:29:52.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:29:52.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:52.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:52 smithi116 ceph-mon[23713]: Reconfiguring daemon mgr.smithi116.poxcrq on smithi116 2024-09-20T19:29:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:52 smithi118 ceph-mon[25856]: Reconfiguring mgr.smithi116.poxcrq (unknown last config time)... 2024-09-20T19:29:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:29:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:29:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:29:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:52 smithi118 ceph-mon[25856]: Reconfiguring daemon mgr.smithi116.poxcrq on smithi116 2024-09-20T19:29:53.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:53 smithi116 ceph-mon[23713]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:53.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:53 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/4235102196' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4191593c-d1a7-414d-916d-cf8e49ebc66a"}]: dispatch 2024-09-20T19:29:53.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:53 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/4235102196' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4191593c-d1a7-414d-916d-cf8e49ebc66a"}]': finished 2024-09-20T19:29:53.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:53 smithi116 ceph-mon[23713]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T19:29:53.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:29:53.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:53 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2864411936' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:29:53.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:53 smithi118 ceph-mon[25856]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:53.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:53 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/4235102196' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4191593c-d1a7-414d-916d-cf8e49ebc66a"}]: dispatch 2024-09-20T19:29:53.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:53 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/4235102196' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4191593c-d1a7-414d-916d-cf8e49ebc66a"}]': finished 2024-09-20T19:29:53.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:53 smithi118 ceph-mon[25856]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T19:29:53.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:29:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:53 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2864411936' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:29:55.687 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:55 smithi116 ceph-mon[23713]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:55.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:55 smithi118 ceph-mon[25856]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:57.834 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:57 smithi116 ceph-mon[23713]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:57.834 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:57.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:57 smithi118 ceph-mon[25856]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:57.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:29:58.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:58 smithi116 ceph-mon[23713]: Reconfiguring alertmanager.smithi116 (dependencies changed)... 2024-09-20T19:29:58.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:58 smithi116 ceph-mon[23713]: Reconfiguring daemon alertmanager.smithi116 on smithi116 2024-09-20T19:29:58.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:58 smithi118 ceph-mon[25856]: Reconfiguring alertmanager.smithi116 (dependencies changed)... 2024-09-20T19:29:58.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:58 smithi118 ceph-mon[25856]: Reconfiguring daemon alertmanager.smithi116 on smithi116 2024-09-20T19:29:59.751 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:29:59 smithi116 ceph-mon[23713]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:29:59.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:29:59 smithi118 ceph-mon[25856]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:00.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:00 smithi116 ceph-mon[23713]: overall HEALTH_OK 2024-09-20T19:30:00.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:00 smithi118 ceph-mon[25856]: overall HEALTH_OK 2024-09-20T19:30:01.778 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:01 smithi116 ceph-mon[23713]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:01.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:01 smithi118 ceph-mon[25856]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:03.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:03 smithi116 ceph-mon[23713]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:03.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:03 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:03.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:03 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T19:30:03.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:03 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:03.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:03 smithi118 ceph-mon[25856]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:03.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:03 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:03.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:03 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T19:30:03.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:03 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:04.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: Reconfiguring crash.smithi116 (monmap changed)... 2024-09-20T19:30:04.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: Reconfiguring daemon crash.smithi116 on smithi116 2024-09-20T19:30:04.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T19:30:04.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:04.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: Deploying daemon osd.0 on smithi116 2024-09-20T19:30:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:30:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:30:04.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: Reconfiguring crash.smithi116 (monmap changed)... 2024-09-20T19:30:04.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: Reconfiguring daemon crash.smithi116 on smithi116 2024-09-20T19:30:04.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T19:30:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: Deploying daemon osd.0 on smithi116 2024-09-20T19:30:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:30:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:30:06.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:06 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:06.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:06 smithi118 ceph-mon[25856]: Reconfiguring grafana.smithi116 (dependencies changed)... 2024-09-20T19:30:06.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:06 smithi118 ceph-mon[25856]: Reconfiguring daemon grafana.smithi116 on smithi116 2024-09-20T19:30:06.467 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:06 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:06.467 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:06 smithi116 ceph-mon[23713]: Reconfiguring grafana.smithi116 (dependencies changed)... 2024-09-20T19:30:06.467 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:06 smithi116 ceph-mon[23713]: Reconfiguring daemon grafana.smithi116 on smithi116 2024-09-20T19:30:07.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:07 smithi118 ceph-mon[25856]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:07.600 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:07 smithi116 ceph-mon[23713]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:09.292 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:09 smithi116 ceph-mon[23713]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:09.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:09 smithi118 ceph-mon[25856]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:11.419 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:11 smithi116 ceph-mon[23713]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:11.419 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:11 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:11.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:11 smithi118 ceph-mon[25856]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:11.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:11 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:13.302 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:13 smithi116 ceph-mon[23713]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:13.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:13 smithi118 ceph-mon[25856]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:14.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:14 smithi116 ceph-mon[23713]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T19:30:14.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:14 smithi118 ceph-mon[25856]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T19:30:15.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T19:30:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T19:30:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:30:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T19:30:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:15.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:15.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T19:30:15.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T19:30:15.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:15.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:30:15.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:15.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T19:30:15.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:16.268 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 0 on host 'smithi116' 2024-09-20T19:30:16.424 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-20T19:30:16.424 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-20T19:30:16.424 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:30:16.425 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: osdmap e8: 1 total, 0 up, 1 in 2024-09-20T19:30:16.425 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:16.425 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:16.425 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:16.425 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:30:16.425 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:30:16.425 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:16.564 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-20T19:30:16.564 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-20T19:30:16.564 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: from='osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:30:16.564 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: osdmap e8: 1 total, 0 up, 1 in 2024-09-20T19:30:16.565 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:16.565 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:16.565 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:16.565 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:30:16.565 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:30:16.565 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:16.864 DEBUG:teuthology.orchestra.run.smithi116:osd.0> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.0.service 2024-09-20T19:30:16.867 INFO:tasks.cephadm:Deploying osd.1 on smithi116 with /dev/vg_nvme/lv_3... 2024-09-20T19:30:16.867 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: Reconfiguring mgr.smithi118.rmiazv (monmap changed)... 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: Reconfiguring daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718] boot 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:17.383 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:17 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: Reconfiguring mgr.smithi118.rmiazv (monmap changed)... 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: Reconfiguring daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: osd.0 [v2:172.21.15.116:6802/314877718,v1:172.21.15.116:6803/314877718] boot 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T19:30:17.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:30:17.444 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:18.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:18 smithi116 ceph-mon[23713]: osdmap e10: 1 total, 1 up, 1 in 2024-09-20T19:30:18.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:18.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:18 smithi116 ceph-mon[23713]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-20T19:30:18.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T19:30:18.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T19:30:18.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:18.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:18 smithi116 ceph-mon[23713]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-20T19:30:18.439 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:18 smithi118 ceph-mon[25856]: osdmap e10: 1 total, 1 up, 1 in 2024-09-20T19:30:18.439 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:18.439 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:18 smithi118 ceph-mon[25856]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-20T19:30:18.439 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T19:30:18.439 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T19:30:18.439 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:18.439 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:18 smithi118 ceph-mon[25856]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.116:9093"}]: dispatch 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.116:3000"}]: dispatch 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.116:9095"}]: dispatch 2024-09-20T19:30:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:19.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:19.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:19.644 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:30:19.663 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi116:vg_nvme/lv_3 2024-09-20T19:30:19.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:19.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T19:30:19.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.116:9093"}]: dispatch 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.116:3000"}]: dispatch 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.116:9095"}]: dispatch 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:19.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:20.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:20 smithi116 ceph-mon[23713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T19:30:20.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:20 smithi116 ceph-mon[23713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.116:9093"}]: dispatch 2024-09-20T19:30:20.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:20 smithi116 ceph-mon[23713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T19:30:20.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:20 smithi116 ceph-mon[23713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.116:3000"}]: dispatch 2024-09-20T19:30:20.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:20 smithi116 ceph-mon[23713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T19:30:20.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:20 smithi116 ceph-mon[23713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.116:9095"}]: dispatch 2024-09-20T19:30:20.674 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:20 smithi118 ceph-mon[25856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T19:30:20.674 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:20 smithi118 ceph-mon[25856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.116:9093"}]: dispatch 2024-09-20T19:30:20.674 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:20 smithi118 ceph-mon[25856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T19:30:20.674 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:20 smithi118 ceph-mon[25856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.116:3000"}]: dispatch 2024-09-20T19:30:20.674 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:20 smithi118 ceph-mon[25856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T19:30:20.674 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:20 smithi118 ceph-mon[25856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.116:9095"}]: dispatch 2024-09-20T19:30:21.713 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:21 smithi116 ceph-mon[23713]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:21.713 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:21.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:21 smithi118 ceph-mon[25856]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:21.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:23.657 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:23 smithi116 ceph-mon[23713]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:23.657 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:23 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:30:23.658 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:23 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:30:23.658 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:23 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:23.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:23 smithi118 ceph-mon[25856]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:23.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:23 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:30:23.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:23 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:30:23.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:23 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:24.494 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:24 smithi116 ceph-mon[23713]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:30:24.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:24 smithi118 ceph-mon[25856]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:30:25.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:25 smithi118 ceph-mon[25856]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:25.846 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:25 smithi116 ceph-mon[23713]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:26.601 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:26 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1499322569' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff52a129-dbee-4594-8492-93d43c872f9d"}]: dispatch 2024-09-20T19:30:26.601 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:26 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1499322569' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff52a129-dbee-4594-8492-93d43c872f9d"}]': finished 2024-09-20T19:30:26.601 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:26 smithi116 ceph-mon[23713]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T19:30:26.601 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:26 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:26.601 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:26 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:26.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:26 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1499322569' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff52a129-dbee-4594-8492-93d43c872f9d"}]: dispatch 2024-09-20T19:30:26.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:26 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1499322569' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff52a129-dbee-4594-8492-93d43c872f9d"}]': finished 2024-09-20T19:30:26.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:26 smithi118 ceph-mon[25856]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T19:30:26.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:26 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:26.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:26 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:27.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:27 smithi116 ceph-mon[23713]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:27.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:27 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3770240436' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:30:27.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:27 smithi118 ceph-mon[25856]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:27.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:27 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3770240436' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:30:29.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:29 smithi116 ceph-mon[23713]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:29.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:29 smithi118 ceph-mon[25856]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:31.293 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:31 smithi116 ceph-mon[23713]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:31.444 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:31 smithi118 ceph-mon[25856]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: Detected new or changed devices on smithi116 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi116 to 12864M 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:32.709 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: Detected new or changed devices on smithi116 2024-09-20T19:30:32.709 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:32.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:30:32.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi116 to 12864M 2024-09-20T19:30:32.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:32.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:32.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:32.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:32.710 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:33.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:33 smithi116 ceph-mon[23713]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:33.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:33.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T19:30:33.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:33.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:33 smithi118 ceph-mon[25856]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:33.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:33.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T19:30:33.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:34.601 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:34 smithi116 ceph-mon[23713]: Deploying daemon osd.1 on smithi116 2024-09-20T19:30:34.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:34 smithi118 ceph-mon[25856]: Deploying daemon osd.1 on smithi116 2024-09-20T19:30:35.514 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:35 smithi116 ceph-mon[23713]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:35.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:35 smithi118 ceph-mon[25856]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:37.613 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:37 smithi116 ceph-mon[23713]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:37.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:37 smithi118 ceph-mon[25856]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:39.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:39 smithi118 ceph-mon[25856]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:39.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:39 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:39.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:39 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:39.735 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:39 smithi116 ceph-mon[23713]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:39.735 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:39 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:39.735 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:39 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:41.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:41 smithi116 ceph-mon[23713]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:41.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:41.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:41.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:41.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:41.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:41 smithi118 ceph-mon[25856]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:41.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:41.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:41.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:41.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:43.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:43 smithi116 ceph-mon[23713]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:43.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:43.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:43 smithi118 ceph-mon[25856]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:43.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:44.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:44 smithi116 ceph-mon[23713]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T19:30:44.474 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 1 on host 'smithi116' 2024-09-20T19:30:44.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:44 smithi118 ceph-mon[25856]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T19:30:45.122 DEBUG:teuthology.orchestra.run.smithi116:osd.1> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.1.service 2024-09-20T19:30:45.125 INFO:tasks.cephadm:Deploying osd.2 on smithi116 with /dev/vg_nvme/lv_2... 2024-09-20T19:30:45.125 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T19:30:45.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-mon[23713]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:45.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-mon[23713]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T19:30:45.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-mon[23713]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T19:30:45.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:45.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-mon[23713]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:30:45.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:45.372 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[38211]: 2024-09-20T19:30:45.224+0000 7fcdcc4bb700 -1 osd.1 0 waiting for initial osdmap 2024-09-20T19:30:45.372 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:30:45 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[38211]: 2024-09-20T19:30:45.227+0000 7fcdc8e56700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:30:45.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:45 smithi118 ceph-mon[25856]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:45.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:45 smithi118 ceph-mon[25856]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T19:30:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:45 smithi118 ceph-mon[25856]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T19:30:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:45 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:45 smithi118 ceph-mon[25856]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:30:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:45 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:46.338 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:46 smithi116 ceph-mon[23713]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:30:46.338 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:46 smithi116 ceph-mon[23713]: osdmap e13: 2 total, 1 up, 2 in 2024-09-20T19:30:46.338 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:46 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:46.338 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:46 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:46.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:46 smithi118 ceph-mon[25856]: from='osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:30:46.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:46 smithi118 ceph-mon[25856]: osdmap e13: 2 total, 1 up, 2 in 2024-09-20T19:30:46.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:46 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:46.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:46 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:47 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:30:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:47 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:30:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:47 smithi118 ceph-mon[25856]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:47 smithi118 ceph-mon[25856]: osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765] boot 2024-09-20T19:30:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:47 smithi118 ceph-mon[25856]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T19:30:47.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:47 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:47.610 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:47 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:30:47.610 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:47 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:30:47.610 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:47 smithi116 ceph-mon[23713]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T19:30:47.610 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:47 smithi116 ceph-mon[23713]: osd.1 [v2:172.21.15.116:6810/3780918765,v1:172.21.15.116:6811/3780918765] boot 2024-09-20T19:30:47.611 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:47 smithi116 ceph-mon[23713]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T19:30:47.611 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:47 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:30:48.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:48 smithi118 ceph-mon[25856]: osdmap e15: 2 total, 2 up, 2 in 2024-09-20T19:30:48.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:48 smithi116 ceph-mon[23713]: osdmap e15: 2 total, 2 up, 2 in 2024-09-20T19:30:49.363 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:49 smithi116 ceph-mon[23713]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:49.368 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:30:49.390 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi116:vg_nvme/lv_2 2024-09-20T19:30:49.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:49 smithi118 ceph-mon[25856]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:50.724 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:50 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:50.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:50 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:51.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:51 smithi116 ceph-mon[23713]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:51.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:51 smithi118 ceph-mon[25856]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:52.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:30:52.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:30:52.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:30:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:30:52.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:53.523 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:53 smithi116 ceph-mon[23713]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:30:53.524 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:53 smithi116 ceph-mon[23713]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:53.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:53 smithi118 ceph-mon[25856]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:30:53.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:53 smithi118 ceph-mon[25856]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:55.096 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:55.096 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:30:55.096 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:30:55.096 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi116 to 6432M 2024-09-20T19:30:55.097 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:55.097 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:55.097 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:55.097 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:55.097 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:55.097 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:54 smithi116 ceph-mon[23713]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:55.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:55.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi116 to 6432M 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:30:55.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:54 smithi118 ceph-mon[25856]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:56.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:55 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3881774724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d517a5f6-72d8-431b-bcd4-ee16374e50f2"}]: dispatch 2024-09-20T19:30:56.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:55 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3881774724' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d517a5f6-72d8-431b-bcd4-ee16374e50f2"}]': finished 2024-09-20T19:30:56.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:55 smithi116 ceph-mon[23713]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T19:30:56.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:55 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:30:56.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:55 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3881774724' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d517a5f6-72d8-431b-bcd4-ee16374e50f2"}]: dispatch 2024-09-20T19:30:56.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:55 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3881774724' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d517a5f6-72d8-431b-bcd4-ee16374e50f2"}]': finished 2024-09-20T19:30:56.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:55 smithi118 ceph-mon[25856]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T19:30:56.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:55 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:30:56.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:56 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2050017039' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:30:56.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:56 smithi116 ceph-mon[23713]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:57.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:56 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2050017039' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:30:57.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:56 smithi118 ceph-mon[25856]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:59.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:59 smithi116 ceph-mon[23713]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:59.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:59 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:59.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:30:59 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:59.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:59 smithi118 ceph-mon[25856]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:30:59.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:59 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:30:59.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:30:59 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:01.257 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:01 smithi116 ceph-mon[23713]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:01.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:01 smithi118 ceph-mon[25856]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:02.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:02 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T19:31:02.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:02 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:02.269 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:02 smithi116 ceph-mon[23713]: Deploying daemon osd.2 on smithi116 2024-09-20T19:31:02.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:02 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T19:31:02.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:02 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:02.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:02 smithi118 ceph-mon[25856]: Deploying daemon osd.2 on smithi116 2024-09-20T19:31:03.183 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:03 smithi116 ceph-mon[23713]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:03.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:03 smithi118 ceph-mon[25856]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:04.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:31:04.267 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:31:04.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:31:04.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:31:05.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:05 smithi116 ceph-mon[23713]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:05.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:05 smithi118 ceph-mon[25856]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:07.864 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:07 smithi116 ceph-mon[23713]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:07.864 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:07.864 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:07.864 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:07.864 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:07.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:07 smithi118 ceph-mon[25856]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:07.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:07.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:09.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:09 smithi116 ceph-mon[23713]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:09.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:09 smithi118 ceph-mon[25856]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:10.280 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 2 on host 'smithi116' 2024-09-20T19:31:10.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:10 smithi116 ceph-mon[23713]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T19:31:10.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:10.945 DEBUG:teuthology.orchestra.run.smithi116:osd.2> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.2.service 2024-09-20T19:31:10.947 INFO:tasks.cephadm:Deploying osd.3 on smithi116 with /dev/vg_nvme/lv_1... 2024-09-20T19:31:10.947 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T19:31:10.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:10 smithi118 ceph-mon[25856]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T19:31:10.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:10 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:11.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:11 smithi116 ceph-mon[23713]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:11.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:11 smithi116 ceph-mon[23713]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T19:31:11.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:11 smithi116 ceph-mon[23713]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T19:31:11.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:11 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:11.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:11 smithi116 ceph-mon[23713]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:31:11.872 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:11 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[41876]: 2024-09-20T19:31:11.579+0000 7f76e4ecf700 -1 osd.2 0 waiting for initial osdmap 2024-09-20T19:31:11.872 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:11 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[41876]: 2024-09-20T19:31:11.583+0000 7f76e186a700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:31:11.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:11 smithi118 ceph-mon[25856]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:11 smithi118 ceph-mon[25856]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T19:31:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:11 smithi118 ceph-mon[25856]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T19:31:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:11 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:11 smithi118 ceph-mon[25856]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:31:12.790 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:12 smithi116 ceph-mon[23713]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:31:12.790 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:12 smithi116 ceph-mon[23713]: osdmap e18: 3 total, 2 up, 3 in 2024-09-20T19:31:12.790 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:12 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:12.790 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:12 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:12.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:12 smithi118 ceph-mon[25856]: from='osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:31:12.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:12 smithi118 ceph-mon[25856]: osdmap e18: 3 total, 2 up, 3 in 2024-09-20T19:31:12.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:12 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:12.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:12 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:13.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:13 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:31:13.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:13 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:31:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:13 smithi116 ceph-mon[23713]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:13 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:13 smithi116 ceph-mon[23713]: osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776] boot 2024-09-20T19:31:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:13 smithi116 ceph-mon[23713]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T19:31:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:13 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:13.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:13 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:31:13.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:13 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:31:13.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:13 smithi118 ceph-mon[25856]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T19:31:13.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:13 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:13.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:13 smithi118 ceph-mon[25856]: osd.2 [v2:172.21.15.116:6818/3553629776,v1:172.21.15.116:6819/3553629776] boot 2024-09-20T19:31:13.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:13 smithi118 ceph-mon[25856]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T19:31:13.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:13 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:31:14.194 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:31:14.213 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi116:vg_nvme/lv_1 2024-09-20T19:31:14.861 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:14 smithi116 ceph-mon[23713]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T19:31:14.861 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:14 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T19:31:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:14 smithi118 ceph-mon[25856]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T19:31:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:14 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T19:31:15.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:15 smithi116 ceph-mon[23713]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:15.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T19:31:15.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:15 smithi116 ceph-mon[23713]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T19:31:15.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T19:31:15.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:15 smithi118 ceph-mon[25856]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:15.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T19:31:15.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:15 smithi118 ceph-mon[25856]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T19:31:15.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T19:31:16.540 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43232]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T19:31:16.540 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43232]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:16.540 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43232]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:16.804 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43287]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T19:31:16.804 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43287]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:16.804 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43287]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:16.804 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T19:31:16.804 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:16 smithi116 ceph-mon[23713]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T19:31:16.805 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:16 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:16.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T19:31:16.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:16 smithi118 ceph-mon[25856]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T19:31:16.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:16 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:17.058 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43297]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T19:31:17.058 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43297]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:17.058 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:31:16 smithi116 sudo[43297]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:17.341 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43397]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T19:31:17.341 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43397]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:17.341 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43397]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:17.602 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43445]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T19:31:17.602 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43445]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:17.603 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43445]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:17.855 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43480]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T19:31:17.855 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43480]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:17.855 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43480]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:17.855 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 ceph-mon[23713]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:17.856 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 ceph-mon[23713]: osdmap e23: 3 total, 3 up, 3 in 2024-09-20T19:31:17.856 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:17.856 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:31:17.856 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:31:17.856 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:17.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:17 smithi118 ceph-mon[25856]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:17.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:17 smithi118 ceph-mon[25856]: osdmap e23: 3 total, 3 up, 3 in 2024-09-20T19:31:17.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:17 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:17.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:17 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:31:17.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:17 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:31:17.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:17 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:18.117 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43497]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T19:31:18.117 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43497]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:18.117 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:17 smithi116 sudo[43497]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:18.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 sudo[43519]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T19:31:18.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 sudo[43519]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:18.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 sudo[43519]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:18.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 sudo[28010]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T19:31:18.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 sudo[28010]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:18.614 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 sudo[28010]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:18.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 ceph-mon[23713]: from='client.14290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:31:18.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 ceph-mon[23713]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T19:31:18.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 ceph-mon[23713]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T19:31:18.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:31:18.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:31:18.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:31:18.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:18 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:31:18.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 sudo[28013]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 sudo[28013]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 sudo[28013]: pam_unix(sudo:session): session closed for user root 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 ceph-mon[25856]: from='client.14290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi116:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 ceph-mon[25856]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 ceph-mon[25856]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:31:18.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:18 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:31:19.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:19 smithi116 ceph-mon[23713]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:19.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:19 smithi116 ceph-mon[23713]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T19:31:19.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:19 smithi116 ceph-mon[23713]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T19:31:19.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:19 smithi116 ceph-mon[23713]: mgrmap e19: smithi116.poxcrq(active, since 2m), standbys: smithi118.rmiazv 2024-09-20T19:31:19.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:19.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:19 smithi118 ceph-mon[25856]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:19.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:19 smithi118 ceph-mon[25856]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T19:31:19.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:19 smithi118 ceph-mon[25856]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T19:31:19.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:19 smithi118 ceph-mon[25856]: mgrmap e19: smithi116.poxcrq(active, since 2m), standbys: smithi118.rmiazv 2024-09-20T19:31:19.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:21.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:21 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:22.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:22.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:22.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:22.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:22.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:22.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:22.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:22.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:22.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:22.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:22.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:21 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:22.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:22 smithi118 ceph-mon[25856]: Detected new or changed devices on smithi116 2024-09-20T19:31:22.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:22 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi116 to 4288M 2024-09-20T19:31:22.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:22 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1411408211' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6dce9a74-6394-4ff0-a17d-714a48600082"}]: dispatch 2024-09-20T19:31:22.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:22 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1411408211' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6dce9a74-6394-4ff0-a17d-714a48600082"}]': finished 2024-09-20T19:31:22.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:22 smithi118 ceph-mon[25856]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T19:31:22.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:22 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:22.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:22 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2625370806' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:31:23.023 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:22 smithi116 ceph-mon[23713]: Detected new or changed devices on smithi116 2024-09-20T19:31:23.024 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:22 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi116 to 4288M 2024-09-20T19:31:23.024 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:22 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1411408211' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6dce9a74-6394-4ff0-a17d-714a48600082"}]: dispatch 2024-09-20T19:31:23.024 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:22 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1411408211' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6dce9a74-6394-4ff0-a17d-714a48600082"}]': finished 2024-09-20T19:31:23.024 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:22 smithi116 ceph-mon[23713]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T19:31:23.024 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:22 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:23.024 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:22 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2625370806' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:31:23.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:23 smithi116 ceph-mon[23713]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:23.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:23 smithi118 ceph-mon[25856]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:24.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:24 smithi118 ceph-mon[25856]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-20T19:31:25.016 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:24 smithi116 ceph-mon[23713]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-20T19:31:27.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:27 smithi116 ceph-mon[23713]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-20T19:31:27.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:27 smithi118 ceph-mon[25856]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-20T19:31:29.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:29 smithi116 ceph-mon[23713]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T19:31:29.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T19:31:29.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:29.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:29 smithi116 ceph-mon[23713]: Deploying daemon osd.3 on smithi116 2024-09-20T19:31:29.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:29 smithi118 ceph-mon[25856]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T19:31:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T19:31:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:29 smithi118 ceph-mon[25856]: Deploying daemon osd.3 on smithi116 2024-09-20T19:31:31.276 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:31 smithi116 ceph-mon[23713]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T19:31:31.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:31 smithi118 ceph-mon[25856]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-20T19:31:33.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:33 smithi116 ceph-mon[23713]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 77 KiB/s wr, 5 op/s 2024-09-20T19:31:33.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:33.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:33.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:33.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:33.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:33 smithi118 ceph-mon[25856]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 77 KiB/s wr, 5 op/s 2024-09-20T19:31:33.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:33.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:33.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:33.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:35.348 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:35 smithi116 ceph-mon[23713]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T19:31:35.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:35 smithi118 ceph-mon[25856]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-20T19:31:36.190 INFO:teuthology.orchestra.run.smithi116.stdout:Created osd(s) 3 on host 'smithi116' 2024-09-20T19:31:36.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:36 smithi116 ceph-mon[23713]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T19:31:36.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:36 smithi118 ceph-mon[25856]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T19:31:36.863 DEBUG:teuthology.orchestra.run.smithi116:osd.3> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.3.service 2024-09-20T19:31:36.865 INFO:tasks.cephadm:Deploying osd.4 on smithi118 with /dev/vg_nvme/lv_4... 2024-09-20T19:31:36.865 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T19:31:37.170 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:37 smithi118 ceph-mon[25856]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:37.170 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:37 smithi118 ceph-mon[25856]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T19:31:37.170 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:37 smithi118 ceph-mon[25856]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T19:31:37.170 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:37 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:37.170 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:37 smithi118 ceph-mon[25856]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:31:37.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:37 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-mon[23713]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-mon[23713]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T19:31:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-mon[23713]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T19:31:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-mon[23713]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]: dispatch 2024-09-20T19:31:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:37.622 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[45107]: 2024-09-20T19:31:37.142+0000 7fbec0335700 -1 osd.3 0 waiting for initial osdmap 2024-09-20T19:31:37.622 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:31:37 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[45107]: 2024-09-20T19:31:37.146+0000 7fbebb4cd700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:31:38.330 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:38 smithi118 ceph-mon[25856]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:31:38.330 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:38 smithi118 ceph-mon[25856]: osdmap e26: 4 total, 3 up, 4 in 2024-09-20T19:31:38.331 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:38 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:38.331 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:38 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:38 smithi116 ceph-mon[23713]: from='osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi116", "root=default"]}]': finished 2024-09-20T19:31:38.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:38 smithi116 ceph-mon[23713]: osdmap e26: 4 total, 3 up, 4 in 2024-09-20T19:31:38.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:38 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:38.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:38 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:39.419 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:39 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:31:39.419 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:39 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:31:39.419 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:39 smithi118 ceph-mon[25856]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:39.419 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:39 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:39.419 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:39 smithi118 ceph-mon[25856]: osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829] boot 2024-09-20T19:31:39.420 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:39 smithi118 ceph-mon[25856]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T19:31:39.420 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:39 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:39.420 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:39 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:31:39.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:39 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:31:39.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:39 smithi116 ceph-mon[23713]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T19:31:39.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:39 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:39.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:39 smithi116 ceph-mon[23713]: osd.3 [v2:172.21.15.116:6826/1159062829,v1:172.21.15.116:6827/1159062829] boot 2024-09-20T19:31:39.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:39 smithi116 ceph-mon[23713]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T19:31:39.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:39 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:31:39.424 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:31:39.444 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi118:vg_nvme/lv_4 2024-09-20T19:31:41.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:40 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:41.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:40 smithi118 ceph-mon[25856]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:41.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:40 smithi118 ceph-mon[25856]: osdmap e28: 4 total, 4 up, 4 in 2024-09-20T19:31:41.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:40 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:41.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:40 smithi116 ceph-mon[23713]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:41.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:40 smithi116 ceph-mon[23713]: osdmap e28: 4 total, 4 up, 4 in 2024-09-20T19:31:42.081 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:41 smithi118 ceph-mon[25856]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:31:42.081 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:31:42.082 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:31:42.082 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:42.121 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:41 smithi116 ceph-mon[23713]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:31:42.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:31:42.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:31:42.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:43.213 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:42 smithi118 ceph-mon[25856]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:43.219 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:42 smithi116 ceph-mon[23713]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: Detected new or changed devices on smithi116 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi116 to 3216M 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:44.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/786788152' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec35d142-0154-402f-8e32-213e9a16aa60"}]: dispatch 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec35d142-0154-402f-8e32-213e9a16aa60"}]: dispatch 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ec35d142-0154-402f-8e32-213e9a16aa60"}]': finished 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:31:44.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:44 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2414903093' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:31:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: Detected new or changed devices on smithi116 2024-09-20T19:31:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi116 to 3216M 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/786788152' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec35d142-0154-402f-8e32-213e9a16aa60"}]: dispatch 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec35d142-0154-402f-8e32-213e9a16aa60"}]: dispatch 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ec35d142-0154-402f-8e32-213e9a16aa60"}]': finished 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T19:31:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:31:44.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:44 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/2414903093' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:31:45.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:45 smithi116 ceph-mon[23713]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:45.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:45 smithi118 ceph-mon[25856]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:47.588 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:47 smithi118 ceph-mon[25856]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:47.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:47 smithi116 ceph-mon[23713]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:49.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:49 smithi116 ceph-mon[23713]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T19:31:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:49.878 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:49 smithi118 ceph-mon[25856]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:49.879 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T19:31:49.879 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:50.781 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:50 smithi118 ceph-mon[25856]: Deploying daemon osd.4 on smithi118 2024-09-20T19:31:50.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:50 smithi116 ceph-mon[23713]: Deploying daemon osd.4 on smithi118 2024-09-20T19:31:51.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:51 smithi116 ceph-mon[23713]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:51.929 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:51 smithi118 ceph-mon[25856]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:53.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:53 smithi116 ceph-mon[23713]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:53.932 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:53 smithi118 ceph-mon[25856]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:55.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:55 smithi116 ceph-mon[23713]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:55.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:55 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:55.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:55 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:55.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:55 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:55.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:55 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:55.638 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:55 smithi118 ceph-mon[25856]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:55.639 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:55 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:55.639 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:55 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:31:55.639 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:55 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:31:55.639 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:55 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:31:57.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:57 smithi116 ceph-mon[23713]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:57.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:57 smithi118 ceph-mon[25856]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:57.759 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 4 on host 'smithi118' 2024-09-20T19:31:58.362 DEBUG:teuthology.orchestra.run.smithi118:osd.4> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.4.service 2024-09-20T19:31:58.365 INFO:tasks.cephadm:Deploying osd.5 on smithi118 with /dev/vg_nvme/lv_3... 2024-09-20T19:31:58.365 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T19:31:58.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:58 smithi116 ceph-mon[23713]: from='osd.4 [v2:172.21.15.118:6800/140910419,v1:172.21.15.118:6801/140910419]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T19:31:58.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:58 smithi116 ceph-mon[23713]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T19:31:58.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:58 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:58.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:58 smithi118 ceph-mon[25856]: from='osd.4 [v2:172.21.15.118:6800/140910419,v1:172.21.15.118:6801/140910419]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T19:31:58.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:58 smithi118 ceph-mon[25856]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T19:31:58.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:58 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:59.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:59 smithi116 ceph-mon[23713]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:59.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:59 smithi116 ceph-mon[23713]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T19:31:59.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:59 smithi116 ceph-mon[23713]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T19:31:59.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:59 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:31:59.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:59 smithi116 ceph-mon[23713]: from='osd.4 [v2:172.21.15.118:6800/140910419,v1:172.21.15.118:6801/140910419]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:31:59.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:59 smithi116 ceph-mon[23713]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:31:59.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:31:59 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:59.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-mon[25856]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:31:59.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-mon[25856]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T19:31:59.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-mon[25856]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T19:31:59.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:31:59.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-mon[25856]: from='osd.4 [v2:172.21.15.118:6800/140910419,v1:172.21.15.118:6801/140910419]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:31:59.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-mon[25856]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:31:59.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:31:59.714 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[29917]: 2024-09-20T19:31:59.362+0000 7fbf436fb700 -1 osd.4 0 waiting for initial osdmap 2024-09-20T19:31:59.714 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:31:59 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[29917]: 2024-09-20T19:31:59.366+0000 7fbf3e092700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:32:00.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:00 smithi116 ceph-mon[23713]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:32:00.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:00 smithi116 ceph-mon[23713]: osdmap e31: 5 total, 4 up, 5 in 2024-09-20T19:32:00.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:00 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:32:00.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:00 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:32:00.622 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:00 smithi118 ceph-mon[25856]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:32:00.622 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:00 smithi118 ceph-mon[25856]: osdmap e31: 5 total, 4 up, 5 in 2024-09-20T19:32:00.622 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:00 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:32:00.623 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:00 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:32:01.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:01 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:32:01.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:01 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:32:01.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:01 smithi118 ceph-mon[25856]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:32:01.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:01 smithi118 ceph-mon[25856]: osd.4 [v2:172.21.15.118:6800/140910419,v1:172.21.15.118:6801/140910419] boot 2024-09-20T19:32:01.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:01 smithi118 ceph-mon[25856]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T19:32:01.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:01 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:32:01.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:01 smithi118 ceph-mon[25856]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T19:32:01.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:01 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:32:01.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:01 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:32:01.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:01 smithi116 ceph-mon[23713]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T19:32:01.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:01 smithi116 ceph-mon[23713]: osd.4 [v2:172.21.15.118:6800/140910419,v1:172.21.15.118:6801/140910419] boot 2024-09-20T19:32:01.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:01 smithi116 ceph-mon[23713]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T19:32:01.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:01 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:32:01.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:01 smithi116 ceph-mon[23713]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T19:32:01.908 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:32:01.926 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi118:vg_nvme/lv_3 2024-09-20T19:32:03.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:03 smithi116 ceph-mon[23713]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:03.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:03 smithi116 ceph-mon[23713]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T19:32:03.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:03 smithi116 ceph-mon[23713]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T19:32:03.682 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:03 smithi118 ceph-mon[25856]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:03.682 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:03 smithi118 ceph-mon[25856]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T19:32:03.682 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:03 smithi118 ceph-mon[25856]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T19:32:04.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:32:04.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:32:04.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:32:04.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:32:04.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:04.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:04 smithi116 ceph-mon[23713]: osdmap e36: 5 total, 5 up, 5 in 2024-09-20T19:32:04.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:32:04.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:32:04.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:32:04.712 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:32:04.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:04.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:04 smithi118 ceph-mon[25856]: osdmap e36: 5 total, 5 up, 5 in 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:05.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='client.24163 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:06.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:06 smithi118 ceph-mon[25856]: Detected new or changed devices on smithi118 2024-09-20T19:32:06.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:06 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi118 to 15936M 2024-09-20T19:32:06.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:06 smithi116 ceph-mon[23713]: Detected new or changed devices on smithi118 2024-09-20T19:32:06.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:06 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi118 to 15936M 2024-09-20T19:32:07.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:07 smithi116 ceph-mon[23713]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:07.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:07 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/3145487138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d8dfbd8-fe64-4270-8592-2e3d498261df"}]: dispatch 2024-09-20T19:32:07.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:07 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d8dfbd8-fe64-4270-8592-2e3d498261df"}]: dispatch 2024-09-20T19:32:07.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:07 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d8dfbd8-fe64-4270-8592-2e3d498261df"}]': finished 2024-09-20T19:32:07.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:07 smithi116 ceph-mon[23713]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T19:32:07.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:07.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:07 smithi118 ceph-mon[25856]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:07.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:07 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/3145487138' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d8dfbd8-fe64-4270-8592-2e3d498261df"}]: dispatch 2024-09-20T19:32:07.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:07 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d8dfbd8-fe64-4270-8592-2e3d498261df"}]: dispatch 2024-09-20T19:32:07.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:07 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d8dfbd8-fe64-4270-8592-2e3d498261df"}]': finished 2024-09-20T19:32:07.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:07 smithi118 ceph-mon[25856]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T19:32:07.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:08.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:08 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/1416635007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:32:08.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:08 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/1416635007' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:32:09.642 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:09 smithi118 ceph-mon[25856]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-20T19:32:09.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:09 smithi116 ceph-mon[23713]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-20T19:32:11.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:11 smithi118 ceph-mon[25856]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-20T19:32:11.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:11 smithi116 ceph-mon[23713]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-20T19:32:13.678 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:13 smithi118 ceph-mon[25856]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T19:32:13.678 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:13 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T19:32:13.678 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:13 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:13 smithi116 ceph-mon[23713]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T19:32:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:13 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T19:32:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:13 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:14.401 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:14 smithi118 ceph-mon[25856]: Deploying daemon osd.5 on smithi118 2024-09-20T19:32:14.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:14 smithi116 ceph-mon[23713]: Deploying daemon osd.5 on smithi118 2024-09-20T19:32:15.621 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:15 smithi118 ceph-mon[25856]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:32:15.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:15 smithi116 ceph-mon[23713]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:32:17.654 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:17 smithi118 ceph-mon[25856]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:32:17.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:17 smithi116 ceph-mon[23713]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:32:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:19 smithi116 ceph-mon[23713]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 46 KiB/s, 0 objects/s recovering 2024-09-20T19:32:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:19.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:19 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:19.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:19 smithi118 ceph-mon[25856]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 46 KiB/s, 0 objects/s recovering 2024-09-20T19:32:19.700 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:19.700 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:19.700 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:19.700 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:19 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:21 smithi118 ceph-mon[25856]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:21.539 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 5 on host 'smithi118' 2024-09-20T19:32:21.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:21 smithi116 ceph-mon[23713]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:22.226 DEBUG:teuthology.orchestra.run.smithi118:osd.5> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.5.service 2024-09-20T19:32:22.228 INFO:tasks.cephadm:Deploying osd.6 on smithi118 with /dev/vg_nvme/lv_2... 2024-09-20T19:32:22.228 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T19:32:22.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:22 smithi118 ceph-mon[25856]: from='osd.5 [v2:172.21.15.118:6808/4007480276,v1:172.21.15.118:6809/4007480276]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T19:32:22.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:22 smithi118 ceph-mon[25856]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T19:32:22.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:22 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:22.621 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:22 smithi116 ceph-mon[23713]: from='osd.5 [v2:172.21.15.118:6808/4007480276,v1:172.21.15.118:6809/4007480276]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T19:32:22.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:22 smithi116 ceph-mon[23713]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T19:32:22.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:22 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-mon[25856]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-mon[25856]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T19:32:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-mon[25856]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T19:32:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-mon[25856]: from='osd.5 [v2:172.21.15.118:6808/4007480276,v1:172.21.15.118:6809/4007480276]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-mon[25856]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:23.464 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[32862]: 2024-09-20T19:32:23.297+0000 7f8f9fb40700 -1 osd.5 0 waiting for initial osdmap 2024-09-20T19:32:23.464 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:32:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[32862]: 2024-09-20T19:32:23.301+0000 7f8f9a4d7700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:32:23.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:23 smithi116 ceph-mon[23713]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:23.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:23 smithi116 ceph-mon[23713]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T19:32:23.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:23 smithi116 ceph-mon[23713]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T19:32:23.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:23 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:23.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:23 smithi116 ceph-mon[23713]: from='osd.5 [v2:172.21.15.118:6808/4007480276,v1:172.21.15.118:6809/4007480276]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:23.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:23 smithi116 ceph-mon[23713]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: osdmap e39: 6 total, 5 up, 6 in 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: osd.5 [v2:172.21.15.118:6808/4007480276,v1:172.21.15.118:6809/4007480276] boot 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T19:32:24.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:24 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: osdmap e39: 6 total, 5 up, 6 in 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: osd.5 [v2:172.21.15.118:6808/4007480276,v1:172.21.15.118:6809/4007480276] boot 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T19:32:24.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:24 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:32:25.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:25 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:32:25.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:25 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:32:25.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:25 smithi118 ceph-mon[25856]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:25.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:25 smithi118 ceph-mon[25856]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T19:32:25.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:25 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:32:25.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:25 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:32:25.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:25 smithi116 ceph-mon[23713]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T19:32:25.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:25 smithi116 ceph-mon[23713]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T19:32:26.565 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:32:26.590 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi118:vg_nvme/lv_2 2024-09-20T19:32:27.533 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:27 smithi118 ceph-mon[25856]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:27.533 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:27 smithi118 ceph-mon[25856]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T19:32:27.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:27 smithi116 ceph-mon[23713]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:27.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:27 smithi116 ceph-mon[23713]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T19:32:28.424 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:28 smithi118 ceph-mon[25856]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T19:32:28.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:28 smithi116 ceph-mon[23713]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: pgmap v133: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: osdmap e44: 6 total, 6 up, 6 in 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:29.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:29 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: pgmap v133: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: osdmap e44: 6 total, 6 up, 6 in 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:29.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:29.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:29 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:30.581 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:30 smithi118 ceph-mon[25856]: Detected new or changed devices on smithi118 2024-09-20T19:32:30.582 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:30 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi118 to 7968M 2024-09-20T19:32:30.582 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:30 smithi118 ceph-mon[25856]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:32:30.582 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:30 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:32:30.582 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:30 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:32:30.582 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:30 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:30.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:30 smithi116 ceph-mon[23713]: Detected new or changed devices on smithi118 2024-09-20T19:32:30.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:30 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi118 to 7968M 2024-09-20T19:32:30.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:30 smithi116 ceph-mon[23713]: from='client.24181 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:32:30.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:30 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:32:30.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:30 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:32:30.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:30 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:31.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:31 smithi116 ceph-mon[23713]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:31.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:31 smithi116 ceph-mon[23713]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T19:32:31.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:31 smithi116 ceph-mon[23713]: Cluster is now healthy 2024-09-20T19:32:31.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:31 smithi118 ceph-mon[25856]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:31.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:31 smithi118 ceph-mon[25856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-20T19:32:31.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:31 smithi118 ceph-mon[25856]: Cluster is now healthy 2024-09-20T19:32:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:32 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2655499514' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db565bb5-3edc-4f92-9192-1540cf62bc72"}]: dispatch 2024-09-20T19:32:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:32 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db565bb5-3edc-4f92-9192-1540cf62bc72"}]: dispatch 2024-09-20T19:32:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:32 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db565bb5-3edc-4f92-9192-1540cf62bc72"}]': finished 2024-09-20T19:32:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:32 smithi116 ceph-mon[23713]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T19:32:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:32 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:32.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:32 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2583225220' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:32:32.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:32 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/2655499514' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db565bb5-3edc-4f92-9192-1540cf62bc72"}]: dispatch 2024-09-20T19:32:32.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:32 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "db565bb5-3edc-4f92-9192-1540cf62bc72"}]: dispatch 2024-09-20T19:32:32.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:32 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "db565bb5-3edc-4f92-9192-1540cf62bc72"}]': finished 2024-09-20T19:32:32.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:32 smithi118 ceph-mon[25856]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T19:32:32.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:32 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:32.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:32 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/2583225220' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:32:33.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:33 smithi116 ceph-mon[23713]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:33.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:33 smithi118 ceph-mon[25856]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:35.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:35 smithi116 ceph-mon[23713]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-20T19:32:35.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:35 smithi118 ceph-mon[25856]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-20T19:32:37.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:37 smithi116 ceph-mon[23713]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T19:32:37.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:37 smithi118 ceph-mon[25856]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T19:32:38.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:38 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T19:32:38.603 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:38 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:38.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:38 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T19:32:38.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:38 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:39.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:39 smithi116 ceph-mon[23713]: Deploying daemon osd.6 on smithi118 2024-09-20T19:32:39.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:39 smithi116 ceph-mon[23713]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-20T19:32:39.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:39 smithi118 ceph-mon[25856]: Deploying daemon osd.6 on smithi118 2024-09-20T19:32:39.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:39 smithi118 ceph-mon[25856]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-20T19:32:41.688 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:41 smithi118 ceph-mon[25856]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:41.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:41 smithi116 ceph-mon[23713]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:43.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:43 smithi116 ceph-mon[23713]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:43.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:43.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:43.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:43.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:43.952 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:43 smithi118 ceph-mon[25856]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:43.952 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:43.952 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:43.952 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:43.952 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:45.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:45 smithi116 ceph-mon[23713]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:45.913 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:45 smithi118 ceph-mon[25856]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:45.944 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 6 on host 'smithi118' 2024-09-20T19:32:46.485 DEBUG:teuthology.orchestra.run.smithi118:osd.6> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.6.service 2024-09-20T19:32:46.488 INFO:tasks.cephadm:Deploying osd.7 on smithi118 with /dev/vg_nvme/lv_1... 2024-09-20T19:32:46.488 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T19:32:46.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:46 smithi116 ceph-mon[23713]: from='osd.6 [v2:172.21.15.118:6816/2305179402,v1:172.21.15.118:6817/2305179402]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T19:32:46.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:46 smithi116 ceph-mon[23713]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T19:32:46.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:46 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:46.935 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:46 smithi118 ceph-mon[25856]: from='osd.6 [v2:172.21.15.118:6816/2305179402,v1:172.21.15.118:6817/2305179402]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T19:32:46.935 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:46 smithi118 ceph-mon[25856]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T19:32:46.935 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:46 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:47.812 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-mon[25856]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:47.812 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-mon[25856]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T19:32:47.812 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-mon[25856]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T19:32:47.812 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:47.812 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-mon[25856]: from='osd.6 [v2:172.21.15.118:6816/2305179402,v1:172.21.15.118:6817/2305179402]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:47.812 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-mon[25856]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:47.812 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[35895]: 2024-09-20T19:32:47.540+0000 7f601f5c0700 -1 osd.6 0 waiting for initial osdmap 2024-09-20T19:32:47.812 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:32:47 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[35895]: 2024-09-20T19:32:47.547+0000 7f601bf5b700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:32:47.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:47 smithi116 ceph-mon[23713]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:47.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:47 smithi116 ceph-mon[23713]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T19:32:47.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:47 smithi116 ceph-mon[23713]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T19:32:47.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:47 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:47.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:47 smithi116 ceph-mon[23713]: from='osd.6 [v2:172.21.15.118:6816/2305179402,v1:172.21.15.118:6817/2305179402]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:47.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:47 smithi116 ceph-mon[23713]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:32:48.694 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:48 smithi118 ceph-mon[25856]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:32:48.694 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:48 smithi118 ceph-mon[25856]: osdmap e47: 7 total, 6 up, 7 in 2024-09-20T19:32:48.694 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:48 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:48.694 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:48 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:48.694 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:48 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:48.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:48 smithi116 ceph-mon[23713]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:32:48.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:48 smithi116 ceph-mon[23713]: osdmap e47: 7 total, 6 up, 7 in 2024-09-20T19:32:48.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:48 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:48.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:48 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:48.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:48 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:49 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:32:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:49 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:32:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:49 smithi116 ceph-mon[23713]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:49 smithi116 ceph-mon[23713]: osd.6 [v2:172.21.15.118:6816/2305179402,v1:172.21.15.118:6817/2305179402] boot 2024-09-20T19:32:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:49 smithi116 ceph-mon[23713]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T19:32:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:49 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:32:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:49 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:32:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:49 smithi118 ceph-mon[25856]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T19:32:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:49 smithi118 ceph-mon[25856]: osd.6 [v2:172.21.15.118:6816/2305179402,v1:172.21.15.118:6817/2305179402] boot 2024-09-20T19:32:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:49 smithi118 ceph-mon[25856]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T19:32:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:32:50.190 INFO:teuthology.orchestra.run.smithi118.stdout: 2024-09-20T19:32:50.213 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi118:vg_nvme/lv_1 2024-09-20T19:32:50.806 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:50 smithi118 ceph-mon[25856]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T19:32:50.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:50 smithi116 ceph-mon[23713]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T19:32:51.696 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:51 smithi118 ceph-mon[25856]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:32:51.696 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:51 smithi118 ceph-mon[25856]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T19:32:51.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:51 smithi116 ceph-mon[23713]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:32:51.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:51 smithi116 ceph-mon[23713]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T19:32:52.692 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:52 smithi118 ceph-mon[25856]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T19:32:52.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:52 smithi116 ceph-mon[23713]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: Detected new or changed devices on smithi118 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: osdmap e52: 7 total, 7 up, 7 in 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi118 to 5312M 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:53.663 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:53.664 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.664 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:32:53.664 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:32:53.664 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:53.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:32:53.910 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: Detected new or changed devices on smithi118 2024-09-20T19:32:53.910 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: osdmap e52: 7 total, 7 up, 7 in 2024-09-20T19:32:53.910 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.910 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:53.910 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:53.910 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi118 to 5312M 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T19:32:53.911 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T19:32:53.912 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:32:54.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:54 smithi116 ceph-mon[23713]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:32:54.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:54 smithi118 ceph-mon[25856]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi118:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:32:55.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:55 smithi116 ceph-mon[23713]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:32:55.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:55 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2003279880' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "020311f1-5844-4c30-be8d-9c3e5f93ea6f"}]: dispatch 2024-09-20T19:32:55.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:55 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "020311f1-5844-4c30-be8d-9c3e5f93ea6f"}]: dispatch 2024-09-20T19:32:55.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:55 smithi116 ceph-mon[23713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "020311f1-5844-4c30-be8d-9c3e5f93ea6f"}]': finished 2024-09-20T19:32:55.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:55 smithi116 ceph-mon[23713]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T19:32:55.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:55 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:32:55.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:55 smithi118 ceph-mon[25856]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:32:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:55 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/2003279880' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "020311f1-5844-4c30-be8d-9c3e5f93ea6f"}]: dispatch 2024-09-20T19:32:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:55 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "020311f1-5844-4c30-be8d-9c3e5f93ea6f"}]: dispatch 2024-09-20T19:32:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:55 smithi118 ceph-mon[25856]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "020311f1-5844-4c30-be8d-9c3e5f93ea6f"}]': finished 2024-09-20T19:32:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:55 smithi118 ceph-mon[25856]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T19:32:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:55 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:32:56.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:56 smithi116 ceph-mon[23713]: from='client.? 172.21.15.118:0/2436411208' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:32:56.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:56 smithi118 ceph-mon[25856]: from='client.? 172.21.15.118:0/2436411208' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T19:32:57.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:57 smithi116 ceph-mon[23713]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T19:32:57.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:57 smithi118 ceph-mon[25856]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-09-20T19:32:59.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:32:59 smithi118 ceph-mon[25856]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-20T19:32:59.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:32:59 smithi116 ceph-mon[23713]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-20T19:33:01.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:01 smithi118 ceph-mon[25856]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T19:33:01.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:01 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T19:33:01.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:01 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:33:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:01 smithi116 ceph-mon[23713]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T19:33:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:01 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T19:33:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:01 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:33:02.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:02 smithi116 ceph-mon[23713]: Deploying daemon osd.7 on smithi118 2024-09-20T19:33:02.890 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:02 smithi118 ceph-mon[25856]: Deploying daemon osd.7 on smithi118 2024-09-20T19:33:03.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:03 smithi116 ceph-mon[23713]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-20T19:33:03.891 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:03 smithi118 ceph-mon[25856]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-20T19:33:04.871 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:33:04.872 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:33:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:33:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:33:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:05 smithi118 ceph-mon[25856]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:33:05.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:05 smithi116 ceph-mon[23713]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:33:07.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:07 smithi116 ceph-mon[23713]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:07.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:07.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:33:07.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:33:07.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:07 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:33:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:07 smithi118 ceph-mon[25856]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:33:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:33:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:07 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:33:09.735 INFO:teuthology.orchestra.run.smithi118.stdout:Created osd(s) 7 on host 'smithi118' 2024-09-20T19:33:09.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:09 smithi116 ceph-mon[23713]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:09.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:09 smithi118 ceph-mon[25856]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:10.326 DEBUG:teuthology.orchestra.run.smithi118:osd.7> sudo journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.7.service 2024-09-20T19:33:10.329 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-20T19:33:10.329 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-20T19:33:10.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:10 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:10.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:10 smithi116 ceph-mon[23713]: from='osd.7 [v2:172.21.15.118:6824/4058156053,v1:172.21.15.118:6825/4058156053]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T19:33:10.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:10 smithi116 ceph-mon[23713]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T19:33:10.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:10 smithi116 ceph-mon[23713]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:10.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:10 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:10.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:10 smithi118 ceph-mon[25856]: from='osd.7 [v2:172.21.15.118:6824/4058156053,v1:172.21.15.118:6825/4058156053]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T19:33:10.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:10 smithi118 ceph-mon[25856]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T19:33:10.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:10 smithi118 ceph-mon[25856]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:11.963 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:11 smithi116 ceph-mon[23713]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T19:33:11.963 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:11 smithi116 ceph-mon[23713]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T19:33:11.963 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:11 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:11.963 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:11 smithi116 ceph-mon[23713]: from='osd.7 [v2:172.21.15.118:6824/4058156053,v1:172.21.15.118:6825/4058156053]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:33:11.963 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:11 smithi116 ceph-mon[23713]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:33:11.963 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:11 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:12.045 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-mon[25856]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T19:33:12.046 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-mon[25856]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T19:33:12.046 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:12.046 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-mon[25856]: from='osd.7 [v2:172.21.15.118:6824/4058156053,v1:172.21.15.118:6825/4058156053]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:33:12.046 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-mon[25856]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]: dispatch 2024-09-20T19:33:12.046 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:12.046 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[38873]: 2024-09-20T19:33:11.743+0000 7f6c5114d700 -1 osd.7 0 waiting for initial osdmap 2024-09-20T19:33:12.046 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:33:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[38873]: 2024-09-20T19:33:11.750+0000 7f6c49ae0700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:33:12.357 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:33:12.986 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:12 smithi116 ceph-mon[23713]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:33:12.987 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:12 smithi116 ceph-mon[23713]: osdmap e55: 8 total, 7 up, 8 in 2024-09-20T19:33:12.987 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:12 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:12.987 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:12 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:12.987 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:12 smithi116 ceph-mon[23713]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:12.987 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:12 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2139317971' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T19:33:13.010 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":55,"num_osds":8,"num_up_osds":7,"osd_up_since":1726860768,"num_in_osds":8,"osd_in_since":1726860775,"num_remapped_pgs":0} 2024-09-20T19:33:13.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:12 smithi118 ceph-mon[25856]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]}]': finished 2024-09-20T19:33:13.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:12 smithi118 ceph-mon[25856]: osdmap e55: 8 total, 7 up, 8 in 2024-09-20T19:33:13.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:12 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:13.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:12 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:13.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:12 smithi118 ceph-mon[25856]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T19:33:13.074 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:12 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2139317971' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T19:33:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:13 smithi116 ceph-mon[23713]: purged_snaps scrub starts 2024-09-20T19:33:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:13 smithi116 ceph-mon[23713]: purged_snaps scrub ok 2024-09-20T19:33:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:13 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:13 smithi116 ceph-mon[23713]: osd.7 [v2:172.21.15.118:6824/4058156053,v1:172.21.15.118:6825/4058156053] boot 2024-09-20T19:33:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:13 smithi116 ceph-mon[23713]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T19:33:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:13 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:13.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:13 smithi116 ceph-mon[23713]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T19:33:14.011 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-20T19:33:14.138 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:13 smithi118 ceph-mon[25856]: purged_snaps scrub starts 2024-09-20T19:33:14.138 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:13 smithi118 ceph-mon[25856]: purged_snaps scrub ok 2024-09-20T19:33:14.138 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:13 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:14.138 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:13 smithi118 ceph-mon[25856]: osd.7 [v2:172.21.15.118:6824/4058156053,v1:172.21.15.118:6825/4058156053] boot 2024-09-20T19:33:14.138 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:13 smithi118 ceph-mon[25856]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T19:33:14.138 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:13 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:33:14.138 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:13 smithi118 ceph-mon[25856]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T19:33:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T19:33:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: Detected new or changed devices on smithi118 2024-09-20T19:33:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi118 to 3984M 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:33:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.469 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: pgmap v170: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: Detected new or changed devices on smithi118 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi118 to 3984M 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:33:15.470 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:33:15.471 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:33:15.957 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:33:16.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:16 smithi118 ceph-mon[25856]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T19:33:16.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:16 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1903726845' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T19:33:16.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:16 smithi116 ceph-mon[23713]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T19:33:16.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:16 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1903726845' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T19:33:16.743 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1726860792,"num_in_osds":8,"osd_in_since":1726860775,"num_remapped_pgs":0} 2024-09-20T19:33:16.744 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T19:33:16.951 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:17.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:17 smithi116 ceph-mon[23713]: pgmap v173: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:17.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:17 smithi116 ceph-mon[23713]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T19:33:17.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:17 smithi118 ceph-mon[25856]: pgmap v173: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:17.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:17 smithi118 ceph-mon[25856]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T19:33:18.584 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:33:18.584 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":60,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","created":"2024-09-20T19:27:18.182688+0000","modified":"2024-09-20T19:33:16.156533+0000","last_up_change":"2024-09-20T19:33:12.741772+0000","last_in_change":"2024-09-20T19:32:55.191586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T19:31:14.301590+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4191593c-d1a7-414d-916d-cf8e49ebc66a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6803","nonce":314877718}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6805","nonce":314877718}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6809","nonce":314877718}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6807","nonce":314877718}]},"public_addr":"172.21.15.116:6803/314877718","cluster_addr":"172.21.15.116:6805/314877718","heartbeat_back_addr":"172.21.15.116:6809/314877718","heartbeat_front_addr":"172.21.15.116:6807/314877718","state":["exists","up"]},{"osd":1,"uuid":"ff52a129-dbee-4594-8492-93d43c872f9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6811","nonce":3780918765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6813","nonce":3780918765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6817","nonce":3780918765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6815","nonce":3780918765}]},"public_addr":"172.21.15.116:6811/3780918765","cluster_addr":"172.21.15.116:6813/3780918765","heartbeat_back_addr":"172.21.15.116:6817/3780918765","heartbeat_front_addr":"172.21.15.116:6815/3780918765","state":["exists","up"]},{"osd":2,"uuid":"d517a5f6-72d8-431b-bcd4-ee16374e50f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6819","nonce":3553629776}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6821","nonce":3553629776}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6825","nonce":3553629776}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6823","nonce":3553629776}]},"public_addr":"172.21.15.116:6819/3553629776","cluster_addr":"172.21.15.116:6821/3553629776","heartbeat_back_addr":"172.21.15.116:6825/3553629776","heartbeat_front_addr":"172.21.15.116:6823/3553629776","state":["exists","up"]},{"osd":3,"uuid":"6dce9a74-6394-4ff0-a17d-714a48600082","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6827","nonce":1159062829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6829","nonce":1159062829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6832","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6833","nonce":1159062829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6831","nonce":1159062829}]},"public_addr":"172.21.15.116:6827/1159062829","cluster_addr":"172.21.15.116:6829/1159062829","heartbeat_back_addr":"172.21.15.116:6833/1159062829","heartbeat_front_addr":"172.21.15.116:6831/1159062829","state":["exists","up"]},{"osd":4,"uuid":"ec35d142-0154-402f-8e32-213e9a16aa60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6800","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6801","nonce":140910419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6802","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6803","nonce":140910419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6806","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6807","nonce":140910419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6804","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6805","nonce":140910419}]},"public_addr":"172.21.15.118:6801/140910419","cluster_addr":"172.21.15.118:6803/140910419","heartbeat_back_addr":"172.21.15.118:6807/140910419","heartbeat_front_addr":"172.21.15.118:6805/140910419","state":["exists","up"]},{"osd":5,"uuid":"0d8dfbd8-fe64-4270-8592-2e3d498261df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6808","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6809","nonce":4007480276}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6810","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6811","nonce":4007480276}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6814","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6815","nonce":4007480276}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6812","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6813","nonce":4007480276}]},"public_addr":"172.21.15.118:6809/4007480276","cluster_addr":"172.21.15.118:6811/4007480276","heartbeat_back_addr":"172.21.15.118:6815/4007480276","heartbeat_front_addr":"172.21.15.118:6813/4007480276","state":["exists","up"]},{"osd":6,"uuid":"db565bb5-3edc-4f92-9192-1540cf62bc72","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6816","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6817","nonce":2305179402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6818","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6819","nonce":2305179402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6822","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6823","nonce":2305179402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6820","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6821","nonce":2305179402}]},"public_addr":"172.21.15.118:6817/2305179402","cluster_addr":"172.21.15.118:6819/2305179402","heartbeat_back_addr":"172.21.15.118:6823/2305179402","heartbeat_front_addr":"172.21.15.118:6821/2305179402","state":["exists","up"]},{"osd":7,"uuid":"020311f1-5844-4c30-be8d-9c3e5f93ea6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6824","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6825","nonce":4058156053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6826","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6827","nonce":4058156053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6830","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6831","nonce":4058156053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6828","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6829","nonce":4058156053}]},"public_addr":"172.21.15.118:6825/4058156053","cluster_addr":"172.21.15.118:6827/4058156053","heartbeat_back_addr":"172.21.15.118:6831/4058156053","heartbeat_front_addr":"172.21.15.118:6829/4058156053","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:30:14.328039+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:30:44.726735+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:10.714884+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:37.071577+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:58.358234+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:32:22.375157+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:32:46.689869+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:33:10.758774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.116:0/1136912864":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/1942267143":"2024-09-21T19:29:03.562232+0000","172.21.15.116:6800/854023151":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/3099128035":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6801/2185164236":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6800/2185164236":"2024-09-21T19:28:13.211308+0000","172.21.15.116:0/2107284629":"2024-09-21T19:28:13.211308+0000","172.21.15.116:0/3423667315":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6801/854023151":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/2690063513":"2024-09-21T19:27:43.458562+0000","172.21.15.116:6800/2973455276":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/874833866":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/3354262598":"2024-09-21T19:29:03.562232+0000","172.21.15.116:6801/2973455276":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/3063702208":"2024-09-21T19:27:43.458562+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T19:33:19.448 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:19 smithi116 ceph-mon[23713]: pgmap v175: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:19.448 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:19 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3029675504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T19:33:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:19 smithi118 ceph-mon[25856]: pgmap v175: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:19 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3029675504' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T19:33:19.472 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-20T19:31:14.301590+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-20T19:33:19.472 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-20T19:33:19.679 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:21 smithi118 ceph-mon[25856]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T19:33:21.574 INFO:teuthology.orchestra.run.smithi116.stdout:pg_num: 1 2024-09-20T19:33:21.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:21 smithi116 ceph-mon[23713]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T19:33:22.243 INFO:tasks.cephadm:Setting up client nodes... 2024-09-20T19:33:22.244 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-20T19:33:22.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:22 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1460893697' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T19:33:22.536 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:22 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1460893697' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T19:33:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:23 smithi118 ceph-mon[25856]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-20T19:33:23.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:23 smithi116 ceph-mon[23713]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-20T19:33:24.315 INFO:teuthology.orchestra.run.smithi116.stdout:[client.0] 2024-09-20T19:33:24.315 INFO:teuthology.orchestra.run.smithi116.stdout: key = AQAEzu1mKoaHEhAAXzh6mOMJmkGnVcObtuOVnQ== 2024-09-20T19:33:25.191 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-09-20T19:33:25.191 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-20T19:33:25.191 DEBUG:teuthology.orchestra.run.smithi116:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-20T19:33:25.228 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph config log 1 --format=json 2024-09-20T19:33:25.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:25 smithi118 ceph-mon[25856]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-20T19:33:25.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:25 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/707503438' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T19:33:25.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:25 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/707503438' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T19:33:25.475 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:25.525 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:25 smithi116 ceph-mon[23713]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-20T19:33:25.526 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:25 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/707503438' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T19:33:25.526 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:25 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/707503438' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T19:33:27.270 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:33:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:27 smithi118 ceph-mon[25856]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:33:27.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:27 smithi116 ceph-mon[23713]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T19:33:27.851 INFO:teuthology.orchestra.run.smithi116.stdout:[{"version":21,"timestamp":"2024-09-20T19:33:14.478906+0000","name":"","changes":[{"name":"osd/host:smithi118/osd_memory_target","previous_value":"5570044450","new_value":"4177533337"}]}] 2024-09-20T19:33:27.852 INFO:tasks.ceph_manager:config epoch is 21 2024-09-20T19:33:27.852 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-20T19:33:27.852 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-20T19:33:27.852 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph mgr dump --format=json 2024-09-20T19:33:28.057 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:28.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:28 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2280888686' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-20T19:33:28.524 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:28 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2280888686' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-20T19:33:29.420 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:29 smithi116 ceph-mon[23713]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-20T19:33:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:29 smithi118 ceph-mon[25856]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-20T19:33:29.884 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:33:30.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:30 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2659966288' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T19:33:30.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:30 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2659966288' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T19:33:30.767 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi116.poxcrq","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6800","nonce":292413321},{"type":"v1","addr":"172.21.15.116:6801","nonce":292413321}]},"active_addr":"172.21.15.116:6801/292413321","active_change":"2024-09-20T19:29:03.562568+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi118.rmiazv","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.116:8443/","prometheus":"http://172.21.15.116:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.116:0","nonce":1951173441}]},{"addrvec":[{"type":"v2","addr":"172.21.15.116:0","nonce":3272549432}]},{"addrvec":[{"type":"v2","addr":"172.21.15.116:0","nonce":2623503467}]},{"addrvec":[{"type":"v2","addr":"172.21.15.116:0","nonce":321025005}]}]}} 2024-09-20T19:33:30.770 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-20T19:33:30.770 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-20T19:33:30.770 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T19:33:30.974 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:31.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:31 smithi116 ceph-mon[23713]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-20T19:33:31.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:31 smithi118 ceph-mon[25856]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-20T19:33:32.738 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:33:32.738 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":60,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","created":"2024-09-20T19:27:18.182688+0000","modified":"2024-09-20T19:33:16.156533+0000","last_up_change":"2024-09-20T19:33:12.741772+0000","last_in_change":"2024-09-20T19:32:55.191586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T19:31:14.301590+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4191593c-d1a7-414d-916d-cf8e49ebc66a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6803","nonce":314877718}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6805","nonce":314877718}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6809","nonce":314877718}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6807","nonce":314877718}]},"public_addr":"172.21.15.116:6803/314877718","cluster_addr":"172.21.15.116:6805/314877718","heartbeat_back_addr":"172.21.15.116:6809/314877718","heartbeat_front_addr":"172.21.15.116:6807/314877718","state":["exists","up"]},{"osd":1,"uuid":"ff52a129-dbee-4594-8492-93d43c872f9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6811","nonce":3780918765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6813","nonce":3780918765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6817","nonce":3780918765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6815","nonce":3780918765}]},"public_addr":"172.21.15.116:6811/3780918765","cluster_addr":"172.21.15.116:6813/3780918765","heartbeat_back_addr":"172.21.15.116:6817/3780918765","heartbeat_front_addr":"172.21.15.116:6815/3780918765","state":["exists","up"]},{"osd":2,"uuid":"d517a5f6-72d8-431b-bcd4-ee16374e50f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6819","nonce":3553629776}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6821","nonce":3553629776}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6825","nonce":3553629776}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6823","nonce":3553629776}]},"public_addr":"172.21.15.116:6819/3553629776","cluster_addr":"172.21.15.116:6821/3553629776","heartbeat_back_addr":"172.21.15.116:6825/3553629776","heartbeat_front_addr":"172.21.15.116:6823/3553629776","state":["exists","up"]},{"osd":3,"uuid":"6dce9a74-6394-4ff0-a17d-714a48600082","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6827","nonce":1159062829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6829","nonce":1159062829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6832","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6833","nonce":1159062829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6831","nonce":1159062829}]},"public_addr":"172.21.15.116:6827/1159062829","cluster_addr":"172.21.15.116:6829/1159062829","heartbeat_back_addr":"172.21.15.116:6833/1159062829","heartbeat_front_addr":"172.21.15.116:6831/1159062829","state":["exists","up"]},{"osd":4,"uuid":"ec35d142-0154-402f-8e32-213e9a16aa60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6800","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6801","nonce":140910419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6802","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6803","nonce":140910419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6806","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6807","nonce":140910419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6804","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6805","nonce":140910419}]},"public_addr":"172.21.15.118:6801/140910419","cluster_addr":"172.21.15.118:6803/140910419","heartbeat_back_addr":"172.21.15.118:6807/140910419","heartbeat_front_addr":"172.21.15.118:6805/140910419","state":["exists","up"]},{"osd":5,"uuid":"0d8dfbd8-fe64-4270-8592-2e3d498261df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6808","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6809","nonce":4007480276}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6810","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6811","nonce":4007480276}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6814","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6815","nonce":4007480276}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6812","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6813","nonce":4007480276}]},"public_addr":"172.21.15.118:6809/4007480276","cluster_addr":"172.21.15.118:6811/4007480276","heartbeat_back_addr":"172.21.15.118:6815/4007480276","heartbeat_front_addr":"172.21.15.118:6813/4007480276","state":["exists","up"]},{"osd":6,"uuid":"db565bb5-3edc-4f92-9192-1540cf62bc72","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6816","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6817","nonce":2305179402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6818","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6819","nonce":2305179402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6822","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6823","nonce":2305179402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6820","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6821","nonce":2305179402}]},"public_addr":"172.21.15.118:6817/2305179402","cluster_addr":"172.21.15.118:6819/2305179402","heartbeat_back_addr":"172.21.15.118:6823/2305179402","heartbeat_front_addr":"172.21.15.118:6821/2305179402","state":["exists","up"]},{"osd":7,"uuid":"020311f1-5844-4c30-be8d-9c3e5f93ea6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6824","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6825","nonce":4058156053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6826","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6827","nonce":4058156053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6830","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6831","nonce":4058156053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6828","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6829","nonce":4058156053}]},"public_addr":"172.21.15.118:6825/4058156053","cluster_addr":"172.21.15.118:6827/4058156053","heartbeat_back_addr":"172.21.15.118:6831/4058156053","heartbeat_front_addr":"172.21.15.118:6829/4058156053","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:30:14.328039+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:30:44.726735+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:10.714884+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:37.071577+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:58.358234+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:32:22.375157+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:32:46.689869+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:33:10.758774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.116:0/1136912864":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/1942267143":"2024-09-21T19:29:03.562232+0000","172.21.15.116:6800/854023151":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/3099128035":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6801/2185164236":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6800/2185164236":"2024-09-21T19:28:13.211308+0000","172.21.15.116:0/2107284629":"2024-09-21T19:28:13.211308+0000","172.21.15.116:0/3423667315":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6801/854023151":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/2690063513":"2024-09-21T19:27:43.458562+0000","172.21.15.116:6800/2973455276":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/874833866":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/3354262598":"2024-09-21T19:29:03.562232+0000","172.21.15.116:6801/2973455276":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/3063702208":"2024-09-21T19:27:43.458562+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T19:33:33.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:33 smithi118 ceph-mon[25856]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:33.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:33 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1049541111' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T19:33:33.473 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:33 smithi116 ceph-mon[23713]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:33.473 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:33 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1049541111' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T19:33:33.498 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-20T19:33:33.498 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T19:33:33.703 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:35 smithi118 ceph-mon[25856]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:35.537 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:33:35.537 INFO:teuthology.orchestra.run.smithi116.stdout:{"epoch":60,"fsid":"2d2015e0-7786-11ef-baf5-efdc52797490","created":"2024-09-20T19:27:18.182688+0000","modified":"2024-09-20T19:33:16.156533+0000","last_up_change":"2024-09-20T19:33:12.741772+0000","last_in_change":"2024-09-20T19:32:55.191586+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T19:31:14.301590+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4191593c-d1a7-414d-916d-cf8e49ebc66a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6802","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6803","nonce":314877718}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6804","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6805","nonce":314877718}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6808","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6809","nonce":314877718}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6806","nonce":314877718},{"type":"v1","addr":"172.21.15.116:6807","nonce":314877718}]},"public_addr":"172.21.15.116:6803/314877718","cluster_addr":"172.21.15.116:6805/314877718","heartbeat_back_addr":"172.21.15.116:6809/314877718","heartbeat_front_addr":"172.21.15.116:6807/314877718","state":["exists","up"]},{"osd":1,"uuid":"ff52a129-dbee-4594-8492-93d43c872f9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6810","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6811","nonce":3780918765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6812","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6813","nonce":3780918765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6816","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6817","nonce":3780918765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6814","nonce":3780918765},{"type":"v1","addr":"172.21.15.116:6815","nonce":3780918765}]},"public_addr":"172.21.15.116:6811/3780918765","cluster_addr":"172.21.15.116:6813/3780918765","heartbeat_back_addr":"172.21.15.116:6817/3780918765","heartbeat_front_addr":"172.21.15.116:6815/3780918765","state":["exists","up"]},{"osd":2,"uuid":"d517a5f6-72d8-431b-bcd4-ee16374e50f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6818","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6819","nonce":3553629776}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6820","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6821","nonce":3553629776}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6824","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6825","nonce":3553629776}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6822","nonce":3553629776},{"type":"v1","addr":"172.21.15.116:6823","nonce":3553629776}]},"public_addr":"172.21.15.116:6819/3553629776","cluster_addr":"172.21.15.116:6821/3553629776","heartbeat_back_addr":"172.21.15.116:6825/3553629776","heartbeat_front_addr":"172.21.15.116:6823/3553629776","state":["exists","up"]},{"osd":3,"uuid":"6dce9a74-6394-4ff0-a17d-714a48600082","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6826","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6827","nonce":1159062829}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6828","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6829","nonce":1159062829}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6832","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6833","nonce":1159062829}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.116:6830","nonce":1159062829},{"type":"v1","addr":"172.21.15.116:6831","nonce":1159062829}]},"public_addr":"172.21.15.116:6827/1159062829","cluster_addr":"172.21.15.116:6829/1159062829","heartbeat_back_addr":"172.21.15.116:6833/1159062829","heartbeat_front_addr":"172.21.15.116:6831/1159062829","state":["exists","up"]},{"osd":4,"uuid":"ec35d142-0154-402f-8e32-213e9a16aa60","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6800","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6801","nonce":140910419}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6802","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6803","nonce":140910419}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6806","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6807","nonce":140910419}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6804","nonce":140910419},{"type":"v1","addr":"172.21.15.118:6805","nonce":140910419}]},"public_addr":"172.21.15.118:6801/140910419","cluster_addr":"172.21.15.118:6803/140910419","heartbeat_back_addr":"172.21.15.118:6807/140910419","heartbeat_front_addr":"172.21.15.118:6805/140910419","state":["exists","up"]},{"osd":5,"uuid":"0d8dfbd8-fe64-4270-8592-2e3d498261df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6808","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6809","nonce":4007480276}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6810","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6811","nonce":4007480276}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6814","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6815","nonce":4007480276}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6812","nonce":4007480276},{"type":"v1","addr":"172.21.15.118:6813","nonce":4007480276}]},"public_addr":"172.21.15.118:6809/4007480276","cluster_addr":"172.21.15.118:6811/4007480276","heartbeat_back_addr":"172.21.15.118:6815/4007480276","heartbeat_front_addr":"172.21.15.118:6813/4007480276","state":["exists","up"]},{"osd":6,"uuid":"db565bb5-3edc-4f92-9192-1540cf62bc72","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6816","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6817","nonce":2305179402}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6818","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6819","nonce":2305179402}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6822","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6823","nonce":2305179402}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6820","nonce":2305179402},{"type":"v1","addr":"172.21.15.118:6821","nonce":2305179402}]},"public_addr":"172.21.15.118:6817/2305179402","cluster_addr":"172.21.15.118:6819/2305179402","heartbeat_back_addr":"172.21.15.118:6823/2305179402","heartbeat_front_addr":"172.21.15.118:6821/2305179402","state":["exists","up"]},{"osd":7,"uuid":"020311f1-5844-4c30-be8d-9c3e5f93ea6f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6824","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6825","nonce":4058156053}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6826","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6827","nonce":4058156053}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6830","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6831","nonce":4058156053}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.118:6828","nonce":4058156053},{"type":"v1","addr":"172.21.15.118:6829","nonce":4058156053}]},"public_addr":"172.21.15.118:6825/4058156053","cluster_addr":"172.21.15.118:6827/4058156053","heartbeat_back_addr":"172.21.15.118:6831/4058156053","heartbeat_front_addr":"172.21.15.118:6829/4058156053","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:30:14.328039+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:30:44.726735+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:10.714884+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:37.071577+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:31:58.358234+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:32:22.375157+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:32:46.689869+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T19:33:10.758774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.116:0/1136912864":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/1942267143":"2024-09-21T19:29:03.562232+0000","172.21.15.116:6800/854023151":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/3099128035":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6801/2185164236":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6800/2185164236":"2024-09-21T19:28:13.211308+0000","172.21.15.116:0/2107284629":"2024-09-21T19:28:13.211308+0000","172.21.15.116:0/3423667315":"2024-09-21T19:28:13.211308+0000","172.21.15.116:6801/854023151":"2024-09-21T19:29:03.562232+0000","172.21.15.116:0/2690063513":"2024-09-21T19:27:43.458562+0000","172.21.15.116:6800/2973455276":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/874833866":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/3354262598":"2024-09-21T19:29:03.562232+0000","172.21.15.116:6801/2973455276":"2024-09-21T19:27:43.458562+0000","172.21.15.116:0/3063702208":"2024-09-21T19:27:43.458562+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T19:33:35.537 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:35 smithi116 ceph-mon[23713]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:36.359 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-20T19:33:36.359 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-20T19:33:36.359 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-20T19:33:36.360 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-20T19:33:36.360 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-20T19:33:36.361 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-20T19:33:36.361 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-20T19:33:36.361 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-20T19:33:36.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:36 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1495065216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T19:33:36.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:36 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1495065216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T19:33:36.749 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:36.749 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:36.786 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:36.786 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:36.788 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:36.795 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:36.795 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:36.976 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:37.365 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:37 smithi118 ceph-mon[25856]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:37 smithi116 ceph-mon[23713]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:39.567 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:39 smithi116 ceph-mon[23713]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:39.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:39 smithi118 ceph-mon[25856]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:41.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:41 smithi116 ceph-mon[23713]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:41.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:41 smithi118 ceph-mon[25856]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:43.611 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:43 smithi116 ceph-mon[23713]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:43 smithi118 ceph-mon[25856]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:45.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:45 smithi116 ceph-mon[23713]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:45 smithi118 ceph-mon[25856]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:47.038 INFO:teuthology.orchestra.run.smithi116.stdout:206158430222 2024-09-20T19:33:47.038 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-20T19:33:47.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:47 smithi118 ceph-mon[25856]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:47.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:47 smithi116 ceph-mon[23713]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:47.590 INFO:teuthology.orchestra.run.smithi116.stdout:171798691859 2024-09-20T19:33:47.590 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-20T19:33:48.363 INFO:teuthology.orchestra.run.smithi116.stdout:81604378658 2024-09-20T19:33:48.364 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-20T19:33:49.338 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:49 smithi116 ceph-mon[23713]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:49.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:49 smithi118 ceph-mon[25856]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:51.191 INFO:teuthology.orchestra.run.smithi116.stdout:115964117022 2024-09-20T19:33:51.192 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-20T19:33:51.554 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:51 smithi116 ceph-mon[23713]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:51.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:51 smithi118 ceph-mon[25856]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:51.986 INFO:teuthology.orchestra.run.smithi116.stdout:137438953496 2024-09-20T19:33:51.986 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-20T19:33:53.246 INFO:teuthology.orchestra.run.smithi116.stdout:38654705709 2024-09-20T19:33:53.247 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-20T19:33:53.472 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:53 smithi116 ceph-mon[23713]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:53.675 INFO:teuthology.orchestra.run.smithi116.stdout:240518168587 2024-09-20T19:33:53.675 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-20T19:33:53.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:53 smithi118 ceph-mon[25856]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:54.124 INFO:teuthology.orchestra.run.smithi116.stdout:60129542184 2024-09-20T19:33:54.124 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-20T19:33:54.187 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:54.187 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:54.188 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:54.188 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:54.190 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:54.194 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:54.194 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:55.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:55 smithi116 ceph-mon[23713]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:55.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:55 smithi118 ceph-mon[25856]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:56.315 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:33:57.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:57 smithi118 ceph-mon[25856]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:57.472 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:57 smithi116 ceph-mon[23713]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:59.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:33:59 smithi116 ceph-mon[23713]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:33:59.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:33:59 smithi118 ceph-mon[25856]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:01.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:01 smithi116 ceph-mon[23713]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:01.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:01 smithi118 ceph-mon[25856]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:02.263 INFO:teuthology.orchestra.run.smithi116.stdout:38654705711 2024-09-20T19:34:02.466 INFO:teuthology.orchestra.run.smithi116.stdout:137438953499 2024-09-20T19:34:03.521 INFO:teuthology.orchestra.run.smithi116.stdout:240518168589 2024-09-20T19:34:03.521 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:03 smithi116 ceph-mon[23713]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:03.521 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:03 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/343061063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T19:34:03.522 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:03 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/158500289' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T19:34:03.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:03 smithi118 ceph-mon[25856]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:03.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:03 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/343061063' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T19:34:03.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:03 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/158500289' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T19:34:04.414 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705711 for osd.0 2024-09-20T19:34:04.414 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:04.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:04 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/4007134714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T19:34:04.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:34:04.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:34:04.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:04 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/4007134714' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T19:34:04.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:34:04.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:34:05.390 INFO:teuthology.orchestra.run.smithi116.stdout:206158430226 2024-09-20T19:34:05.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:05 smithi116 ceph-mon[23713]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:05 smithi118 ceph-mon[25856]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:06.044 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953496 got 137438953499 for osd.4 2024-09-20T19:34:06.044 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:06.275 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:06 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1240534079' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T19:34:06.667 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430226 for osd.6 2024-09-20T19:34:06.667 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:06.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:06 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1240534079' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T19:34:07.026 INFO:teuthology.orchestra.run.smithi116.stdout:60129542187 2024-09-20T19:34:07.256 INFO:teuthology.orchestra.run.smithi116.stdout:81604378662 2024-09-20T19:34:07.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:07 smithi118 ceph-mon[25856]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:07.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:07 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/33299981' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T19:34:07.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:07 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2695324433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T19:34:07.607 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:07 smithi116 ceph-mon[23713]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:07.607 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:07 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/33299981' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T19:34:07.607 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:07 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2695324433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T19:34:07.631 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542187 for osd.1 2024-09-20T19:34:07.631 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:08.234 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168587 got 240518168589 for osd.7 2024-09-20T19:34:08.234 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:08.650 INFO:teuthology.orchestra.run.smithi116.stdout:115964117025 2024-09-20T19:34:09.307 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378662 for osd.2 2024-09-20T19:34:09.308 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:09.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:09 smithi116 ceph-mon[23713]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:09.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:09 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3354023687' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T19:34:09.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:09 smithi118 ceph-mon[25856]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:09.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:09 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3354023687' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T19:34:09.739 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117022 got 115964117025 for osd.3 2024-09-20T19:34:09.739 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:10.341 INFO:teuthology.orchestra.run.smithi116.stdout:171798691864 2024-09-20T19:34:10.861 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691864 for osd.5 2024-09-20T19:34:10.861 DEBUG:teuthology.parallel:result is None 2024-09-20T19:34:10.861 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-20T19:34:10.861 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T19:34:11.068 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:34:11.525 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:11 smithi116 ceph-mon[23713]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:11.525 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:11 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2622869786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T19:34:11.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:11 smithi118 ceph-mon[25856]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:11.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:11 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2622869786' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T19:34:12.866 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:34:12.868 INFO:teuthology.orchestra.run.smithi116.stderr:dumped all 2024-09-20T19:34:13.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:13 smithi116 ceph-mon[23713]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:13.419 INFO:teuthology.orchestra.run.smithi116.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-20T19:34:12.042482+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48992,"kb_used_data":6368,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912224,"statfs":{"total":767960285184,"available":767910117376,"internally_reserved":0,"allocated":6520832,"data_stored":3571156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002043"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-20T19:33:16.166925+0000","last_change":"2024-09-20T19:33:16.166925+0000","last_active":"2024-09-20T19:33:16.166925+0000","last_peered":"2024-09-20T19:33:16.166925+0000","last_clean":"2024-09-20T19:33:16.166925+0000","last_became_active":"2024-09-20T19:33:16.166465+0000","last_became_peered":"2024-09-20T19:33:16.166465+0000","last_unstale":"2024-09-20T19:33:16.166925+0000","last_undegraded":"2024-09-20T19:33:16.166925+0000","last_fullsized":"2024-09-20T19:33:16.166925+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T19:31:14.589457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T19:31:14.589457+0000","last_clean_scrub_stamp":"2024-09-20T19:31:14.589457+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-21T22:21:34.662030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53800000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63100000000000001}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":776342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.53200000000000003}]},{"osd":1,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.504}]},{"osd":2,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.44500000000000001}]},{"osd":3,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.55300000000000005}]},{"osd":4,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.438}]},{"osd":5,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.48199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51600000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.35699999999999998,"15min":0.33700000000000002},"min":{"1min":0.27400000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.371,"15min":0.34100000000000003},"min":{"1min":0.253,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.315}]},{"osd":2,"last update":"Fri Sep 20 19:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.41899999999999998},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.41499999999999998,"15min":0.41199999999999998},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.623,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.26000000000000001}]},{"osd":3,"last update":"Fri Sep 20 19:33:42 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.44700000000000001,"15min":0.45100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.52500000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.438,"5min":0.495,"15min":0.505},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.52700000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.33200000000000002}]},{"osd":4,"last update":"Fri Sep 20 19:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.82099999999999995,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.498},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.66100000000000003,"15min":0.66300000000000003},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.47899999999999998}]},{"osd":5,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.42899999999999999}]},{"osd":6,"last update":"Fri Sep 20 19:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.41699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51300000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 19:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.34200000000000003,"15min":0.32400000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.52100000000000002}]},{"osd":2,"last update":"Fri Sep 20 19:33:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39700000000000002,"15min":0.39400000000000002},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.42299999999999999,"15min":0.41799999999999998},"min":{"1min":0.33300000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.5}]},{"osd":3,"last update":"Fri Sep 20 19:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.434,"15min":0.43099999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.432,"15min":0.42899999999999999},"min":{"1min":0.33100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.54300000000000004}]},{"osd":4,"last update":"Fri Sep 20 19:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.61399999999999999,"15min":0.61599999999999999},"min":{"1min":0.46999999999999997,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.78400000000000003,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.61899999999999999,"15min":0.61699999999999999},"min":{"1min":0.47799999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82799999999999996,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.78500000000000003}]},{"osd":5,"last update":"Fri Sep 20 19:33:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.73399999999999999}]},{"osd":6,"last update":"Fri Sep 20 19:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.747}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81200000000000006}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.38900000000000001,"15min":0.38600000000000001},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.54200000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39200000000000002},"min":{"1min":0.30099999999999999,"5min":0.249,"15min":0.249},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.27900000000000003}]},{"osd":1,"last update":"Fri Sep 20 19:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.41299999999999998,"15min":0.40799999999999997},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57899999999999996,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.253}]},{"osd":3,"last update":"Fri Sep 20 19:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.44600000000000001,"15min":0.45300000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.59199999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.436,"5min":0.45600000000000002,"15min":0.46000000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.33200000000000002}]},{"osd":4,"last update":"Fri Sep 20 19:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58099999999999996,"15min":0.57699999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.316},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.38800000000000001}]},{"osd":5,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.34100000000000003}]},{"osd":6,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.44800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.437}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:41 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40200000000000002,"15min":0.40300000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56699999999999995,"5min":0.623,"15min":0.623},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.436,"15min":0.436},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.41399999999999998}]},{"osd":1,"last update":"Fri Sep 20 19:33:41 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.443,"15min":0.44900000000000001},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.505,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.44}]},{"osd":2,"last update":"Fri Sep 20 19:33:41 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.59499999999999997,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.496},{"interface":"front","average":{"1min":0.439,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60299999999999998,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.45700000000000002}]},{"osd":4,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.41099999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.624},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.501,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78800000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.71899999999999997}]},{"osd":5,"last update":"Fri Sep 20 19:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.72699999999999998}]},{"osd":6,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.73599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77300000000000002}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.51100000000000001,"15min":0.503},"min":{"1min":0.434,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.621},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.56599999999999995,"15min":0.56000000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.47899999999999998}]},{"osd":1,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.51800000000000002,"15min":0.51000000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54600000000000004,"15min":0.54100000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.55400000000000005}]},{"osd":2,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.51600000000000001,"15min":0.51000000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.56200000000000006,"15min":0.55900000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.54600000000000004}]},{"osd":3,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.52700000000000002,"15min":0.51700000000000002},"min":{"1min":0.39700000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.53400000000000003}]},{"osd":5,"last update":"Fri Sep 20 19:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.37},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.379}]},{"osd":6,"last update":"Fri Sep 20 19:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.41599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.49199999999999999}]},{"osd":1,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.55700000000000005}]},{"osd":2,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.43099999999999999}]},{"osd":3,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.53800000000000003}]},{"osd":4,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.57199999999999995}]},{"osd":6,"last update":"Fri Sep 20 19:33:52 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.57999999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T19:34:13.420 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T19:34:13.625 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:34:13.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:13 smithi118 ceph-mon[25856]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:14.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:14 smithi116 ceph-mon[23713]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:14.559 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:14 smithi118 ceph-mon[25856]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:15.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:15 smithi116 ceph-mon[23713]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:15.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:15.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:15.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:15.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:15 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:15.451 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:34:15.453 INFO:teuthology.orchestra.run.smithi116.stderr:dumped all 2024-09-20T19:34:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:15 smithi118 ceph-mon[25856]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:15 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:16.201 INFO:teuthology.orchestra.run.smithi116.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-20T19:34:14.042800+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48992,"kb_used_data":6368,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912224,"statfs":{"total":767960285184,"available":767910117376,"internally_reserved":0,"allocated":6520832,"data_stored":3571156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001932"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-20T19:33:16.166925+0000","last_change":"2024-09-20T19:33:16.166925+0000","last_active":"2024-09-20T19:33:16.166925+0000","last_peered":"2024-09-20T19:33:16.166925+0000","last_clean":"2024-09-20T19:33:16.166925+0000","last_became_active":"2024-09-20T19:33:16.166465+0000","last_became_peered":"2024-09-20T19:33:16.166465+0000","last_unstale":"2024-09-20T19:33:16.166925+0000","last_undegraded":"2024-09-20T19:33:16.166925+0000","last_fullsized":"2024-09-20T19:33:16.166925+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T19:31:14.589457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T19:31:14.589457+0000","last_clean_scrub_stamp":"2024-09-20T19:31:14.589457+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-21T22:21:34.662030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168592,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57799999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59999999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51800000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67200000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":776342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.53200000000000003}]},{"osd":1,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.504}]},{"osd":2,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.44500000000000001}]},{"osd":3,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.55300000000000005}]},{"osd":4,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.438}]},{"osd":5,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.48199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51600000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.35699999999999998,"15min":0.33700000000000002},"min":{"1min":0.27400000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.307},{"interface":"front","average":{"1min":0.438,"5min":0.371,"15min":0.34100000000000003},"min":{"1min":0.253,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.34999999999999998}]},{"osd":2,"last update":"Fri Sep 20 19:33:23 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.41899999999999998},"min":{"1min":0.32800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.432,"5min":0.41499999999999998,"15min":0.41199999999999998},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.623,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.28499999999999998}]},{"osd":3,"last update":"Fri Sep 20 19:33:42 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.44700000000000001,"15min":0.45100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.52500000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.318},{"interface":"front","average":{"1min":0.438,"5min":0.495,"15min":0.505},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.52700000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.34300000000000003}]},{"osd":4,"last update":"Fri Sep 20 19:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.82099999999999995,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.66100000000000003,"15min":0.66300000000000003},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.54500000000000004}]},{"osd":5,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.48399999999999999}]},{"osd":6,"last update":"Fri Sep 20 19:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 19:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.34200000000000003,"15min":0.32400000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.36399999999999999,"15min":0.33800000000000002},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Fri Sep 20 19:33:22 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39700000000000002,"15min":0.39400000000000002},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.42299999999999999,"15min":0.41799999999999998},"min":{"1min":0.33300000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.441}]},{"osd":3,"last update":"Fri Sep 20 19:33:40 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.434,"15min":0.43099999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.499},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.432,"15min":0.42899999999999999},"min":{"1min":0.33100000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42999999999999999}]},{"osd":4,"last update":"Fri Sep 20 19:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.61399999999999999,"15min":0.61599999999999999},"min":{"1min":0.46999999999999997,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.78400000000000003,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.61899999999999999,"15min":0.61699999999999999},"min":{"1min":0.47799999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82799999999999996,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.66600000000000004}]},{"osd":5,"last update":"Fri Sep 20 19:33:28 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.78000000000000003}]},{"osd":6,"last update":"Fri Sep 20 19:33:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.69199999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71399999999999997}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.38900000000000001,"15min":0.38600000000000001},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.54200000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39200000000000002},"min":{"1min":0.30099999999999999,"5min":0.249,"15min":0.249},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.32600000000000001}]},{"osd":1,"last update":"Fri Sep 20 19:33:15 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.41299999999999998,"15min":0.40799999999999997},"min":{"1min":0.32000000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57899999999999996,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.36099999999999999}]},{"osd":3,"last update":"Fri Sep 20 19:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.44600000000000001,"15min":0.45300000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.59199999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.45600000000000002,"15min":0.46000000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.32000000000000001}]},{"osd":4,"last update":"Fri Sep 20 19:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58099999999999996,"15min":0.57699999999999996},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.375},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58599999999999997,"15min":0.58399999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.51100000000000001}]},{"osd":5,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.42999999999999999}]},{"osd":6,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39400000000000002}]}]},{"osd":3,"up_from":27,"seq":115964117026,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:41 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.40200000000000002,"15min":0.40300000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.56699999999999995,"5min":0.623,"15min":0.623},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.436,"15min":0.436},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.41399999999999998}]},{"osd":1,"last update":"Fri Sep 20 19:33:41 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28499999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.443,"15min":0.44900000000000001},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.505,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.44}]},{"osd":2,"last update":"Fri Sep 20 19:33:41 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.59499999999999997,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.496},{"interface":"front","average":{"1min":0.439,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60299999999999998,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.45700000000000002}]},{"osd":4,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.59699999999999998,"15min":0.59899999999999998},"min":{"1min":0.41099999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.624},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.501,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.78800000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.71899999999999997}]},{"osd":5,"last update":"Fri Sep 20 19:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.72699999999999998}]},{"osd":6,"last update":"Fri Sep 20 19:33:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.73599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77300000000000002}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.51100000000000001,"15min":0.503},"min":{"1min":0.434,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.621},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.56599999999999995,"15min":0.56000000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.47899999999999998}]},{"osd":1,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.51800000000000002,"15min":0.51000000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.54600000000000004,"15min":0.54100000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.55400000000000005}]},{"osd":2,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.51600000000000001,"15min":0.51000000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.56200000000000006,"15min":0.55900000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.54600000000000004}]},{"osd":3,"last update":"Fri Sep 20 19:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.52700000000000002,"15min":0.51700000000000002},"min":{"1min":0.39700000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.53400000000000003}]},{"osd":5,"last update":"Fri Sep 20 19:33:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.37},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.379}]},{"osd":6,"last update":"Fri Sep 20 19:33:49 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":1.3100000000000001,"5min":1.3100000000000001,"15min":1.3100000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.41599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.49199999999999999}]},{"osd":1,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.55700000000000005}]},{"osd":2,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.43099999999999999}]},{"osd":3,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.53800000000000003}]},{"osd":4,"last update":"Fri Sep 20 19:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.57199999999999995}]},{"osd":6,"last update":"Fri Sep 20 19:33:52 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.57999999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T19:34:16.202 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-20T19:34:16.202 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-20T19:34:16.202 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-20T19:34:16.203 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph health --format=json 2024-09-20T19:34:16.415 INFO:teuthology.orchestra.run.smithi116.stderr:Inferring config /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/mon.smithi116/config 2024-09-20T19:34:16.536 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:16 smithi116 ceph-mon[23713]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:16.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:16 smithi118 ceph-mon[25856]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:17.476 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:17 smithi116 ceph-mon[23713]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:17.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:17 smithi118 ceph-mon[25856]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:18.184 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:34:18.184 INFO:teuthology.orchestra.run.smithi116.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-20T19:34:18.784 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-20T19:34:18.784 INFO:tasks.cephadm:Setup complete, yielding 2024-09-20T19:34:18.784 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T19:34:18.792 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:34:18.792 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph orch status' 2024-09-20T19:34:19.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:19 smithi116 ceph-mon[23713]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:19.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:19 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2641294319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T19:34:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:19 smithi118 ceph-mon[25856]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:19 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2641294319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T19:34:20.726 INFO:teuthology.orchestra.run.smithi116.stdout:Backend: cephadm 2024-09-20T19:34:20.726 INFO:teuthology.orchestra.run.smithi116.stdout:Available: Yes 2024-09-20T19:34:20.726 INFO:teuthology.orchestra.run.smithi116.stdout:Paused: No 2024-09-20T19:34:21.318 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ps' 2024-09-20T19:34:21.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:21 smithi116 ceph-mon[23713]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:21 smithi118 ceph-mon[25856]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:22.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:22 smithi118 ceph-mon[25856]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:22.611 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:22 smithi116 ceph-mon[23713]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:23.291 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:23 smithi116 ceph-mon[23713]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:23.291 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:34:23.291 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (4m) 2m ago 5m 15.4M - ba2b418f427c 2e1af9a638b9 2024-09-20T19:34:23.291 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (5m) 2m ago 5m 6413k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:34:23.291 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (5m) 71s ago 5m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:34:23.291 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (4m) 2m ago 5m 42.9M - 8.3.5 dad864ee21e9 564eedfb2837 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 401b463eca11 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 running (5m) 71s ago 5m 411M - 17.2.0 e1d6a67b021e 6154e3367da1 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (7m) 2m ago 7m 40.9M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (5m) 71s ago 5m 38.5M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (5m) 2m ago 5m 18.9M - 1dbe0e931976 b18146bfd761 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (4m) 71s ago 4m 17.2M - 1dbe0e931976 2e0ae7aea427 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (4m) 2m ago 4m 35.0M 3216M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (3m) 2m ago 3m 37.1M 3216M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (3m) 2m ago 3m 32.6M 3216M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (2m) 2m ago 2m 11.0M 3216M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (2m) 71s ago 2m 33.3M 3984M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (2m) 71s ago 2m 33.3M 3984M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (101s) 71s ago 100s 30.0M 3984M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (77s) 71s ago 76s 10.6M 3984M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:34:23.292 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (4m) 2m ago 4m 56.0M - 514e6a882f6e b631805c5975 2024-09-20T19:34:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:23 smithi118 ceph-mon[25856]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:23.951 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ls' 2024-09-20T19:34:24.437 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:24 smithi116 ceph-mon[23713]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:24.463 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:24 smithi118 ceph-mon[25856]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:25.452 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:25 smithi116 ceph-mon[23713]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:25.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:25 smithi118 ceph-mon[25856]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:crash 2/2 2m ago 6m * 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:mgr 2/2 2m ago 6m count:2 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:mon 2/2 2m ago 5m smithi116:172.21.15.116=smithi116;smithi118:172.21.15.118=smithi118;count:2 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-20T19:34:25.859 INFO:teuthology.orchestra.run.smithi116.stdout:osd 8 2m ago - 2024-09-20T19:34:25.860 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-20T19:34:26.505 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph orch host ls' 2024-09-20T19:34:27.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:27 smithi116 ceph-mon[23713]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:27.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:27 smithi116 ceph-mon[23713]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:27 smithi118 ceph-mon[25856]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:27 smithi118 ceph-mon[25856]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:28.474 INFO:teuthology.orchestra.run.smithi116.stdout:HOST ADDR LABELS STATUS 2024-09-20T19:34:28.475 INFO:teuthology.orchestra.run.smithi116.stdout:smithi116 172.21.15.116 2024-09-20T19:34:28.475 INFO:teuthology.orchestra.run.smithi116.stdout:smithi118 172.21.15.118 2024-09-20T19:34:28.475 INFO:teuthology.orchestra.run.smithi116.stdout:2 hosts in cluster 2024-09-20T19:34:29.025 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph orch device ls' 2024-09-20T19:34:29.441 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:29 smithi116 ceph-mon[23713]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:29.441 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:29 smithi116 ceph-mon[23713]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:29 smithi118 ceph-mon[25856]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:29 smithi118 ceph-mon[25856]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:30.963 INFO:teuthology.orchestra.run.smithi116.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-20T19:34:30.964 INFO:teuthology.orchestra.run.smithi116.stdout:smithi116 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400VJ400BGN 400G LVM detected, locked 2024-09-20T19:34:30.964 INFO:teuthology.orchestra.run.smithi116.stdout:smithi118 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB747500JS480DGN 480G LVM detected, locked 2024-09-20T19:34:31.282 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:31 smithi116 ceph-mon[23713]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:31.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:31 smithi118 ceph-mon[25856]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:31.641 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T19:34:31.653 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:34:31.654 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-20T19:34:31.681 INFO:teuthology.orchestra.run.smithi116.stderr:+ systemctl stop nfs-server 2024-09-20T19:34:31.689 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi118.front.sepia.ceph.com 2024-09-20T19:34:31.689 DEBUG:teuthology.orchestra.run.smithi118:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-20T19:34:31.718 INFO:teuthology.orchestra.run.smithi118.stderr:+ systemctl stop nfs-server 2024-09-20T19:34:31.727 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T19:34:31.735 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:34:31.735 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph fs volume create foofs' 2024-09-20T19:34:32.335 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:32 smithi116 ceph-mon[23713]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:32.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:32 smithi118 ceph-mon[25856]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:33.227 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:33 smithi116 ceph-mon[23713]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:33.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:33 smithi118 ceph-mon[25856]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:34.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:34 smithi118 ceph-mon[25856]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:34.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:34 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-20T19:34:34.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:34 smithi116 ceph-mon[23713]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:34.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:34 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-20T19:34:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:35 smithi118 ceph-mon[25856]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:35 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-20T19:34:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:35 smithi118 ceph-mon[25856]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T19:34:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:35 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-20T19:34:35.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:35 smithi116 ceph-mon[23713]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:35.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:35 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-20T19:34:35.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:35 smithi116 ceph-mon[23713]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T19:34:35.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:35 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-20T19:34:35.529 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:35 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[23708]: 2024-09-20T19:34:35.212+0000 7f211df78700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:34:35.911 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-20T19:34:35.927 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-20T19:34:35.927 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T19:34:36.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-20T19:34:36.573 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T19:34:36.573 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: fsmap foofs:0 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: Saving service mds.foofs spec with placement count:2 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:36.574 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T19:34:36.575 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T19:34:36.575 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:36.575 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:36 smithi118 ceph-mon[25856]: Deploying daemon mds.foofs.smithi116.xvfgaf on smithi116 2024-09-20T19:34:36.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-20T19:34:36.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T19:34:36.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: fsmap foofs:0 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: Saving service mds.foofs spec with placement count:2 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:36.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:36.577 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T19:34:36.577 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T19:34:36.577 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:36.577 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:36 smithi116 ceph-mon[23713]: Deploying daemon mds.foofs.smithi116.xvfgaf on smithi116 2024-09-20T19:34:37.365 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:37 smithi118 ceph-mon[25856]: pgmap v217: 65 pgs: 10 creating+peering, 1 active+clean, 54 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:37.365 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:37 smithi118 ceph-mon[25856]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T19:34:37.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:37 smithi116 ceph-mon[23713]: pgmap v217: 65 pgs: 10 creating+peering, 1 active+clean, 54 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:37.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:37 smithi116 ceph-mon[23713]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T19:34:38.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:38 smithi116 ceph-mon[23713]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T19:34:38.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:38 smithi118 ceph-mon[25856]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T19:34:39.257 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:34:39.257 INFO:teuthology.orchestra.run.smithi116.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T19:28:02.827016Z", "last_refresh": "2024-09-20T19:31:39.952114Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T19:27:56.093943Z", "last_refresh": "2024-09-20T19:31:39.952176Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T19:27:59.549128Z", "last_refresh": "2024-09-20T19:31:39.952235Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T19:34:35.230375Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T19:34:35.225490Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T19:27:54.712090Z", "last_refresh": "2024-09-20T19:31:39.952028Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi116:172.21.15.116=smithi116", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T19:28:45.778166Z", "last_refresh": "2024-09-20T19:31:39.951888Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T19:28:01.506994Z", "last_refresh": "2024-09-20T19:31:39.952293Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T19:31:39.952412Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T19:27:57.573016Z", "last_refresh": "2024-09-20T19:31:39.952352Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T19:34:39.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:39 smithi116 ceph-mon[23713]: pgmap v220: 65 pgs: 10 creating+peering, 23 active+clean, 32 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:39.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:39 smithi118 ceph-mon[25856]: pgmap v220: 65 pgs: 10 creating+peering, 23 active+clean, 32 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:39.853 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-20T19:34:40.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:40 smithi116 ceph-mon[23713]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:40.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:40 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:40.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:40 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T19:34:40.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:40 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T19:34:40.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:40 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:40.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:40 smithi118 ceph-mon[25856]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:40.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:40 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:40.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:40 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T19:34:40.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:40 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T19:34:40.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:40 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:40.854 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T19:34:41.477 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:41.478 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: Deploying daemon mds.foofs.smithi118.zumcyi on smithi118 2024-09-20T19:34:41.479 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: mds.? [v2:172.21.15.116:6834/1570267385,v1:172.21.15.116:6835/1570267385] up:boot 2024-09-20T19:34:41.479 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: daemon mds.foofs.smithi116.xvfgaf assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-20T19:34:41.480 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T19:34:41.480 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-20T19:34:41.480 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: Cluster is now healthy 2024-09-20T19:34:41.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: fsmap foofs:0 1 up:standby 2024-09-20T19:34:41.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"}]: dispatch 2024-09-20T19:34:41.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:creating} 2024-09-20T19:34:41.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:41 smithi116 ceph-mon[23713]: daemon mds.foofs.smithi116.xvfgaf is now active in filesystem foofs as rank 0 2024-09-20T19:34:41.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T19:34:41.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: Deploying daemon mds.foofs.smithi118.zumcyi on smithi118 2024-09-20T19:34:41.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: mds.? [v2:172.21.15.116:6834/1570267385,v1:172.21.15.116:6835/1570267385] up:boot 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: daemon mds.foofs.smithi116.xvfgaf assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: Cluster is now healthy 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: fsmap foofs:0 1 up:standby 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"}]: dispatch 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:creating} 2024-09-20T19:34:41.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:41 smithi118 ceph-mon[25856]: daemon mds.foofs.smithi116.xvfgaf is now active in filesystem foofs as rank 0 2024-09-20T19:34:42.456 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:42 smithi116 ceph-mon[23713]: mds.? [v2:172.21.15.116:6834/1570267385,v1:172.21.15.116:6835/1570267385] up:active 2024-09-20T19:34:42.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:42 smithi116 ceph-mon[23713]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 2024-09-20T19:34:42.541 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:42 smithi118 ceph-mon[25856]: mds.? [v2:172.21.15.116:6834/1570267385,v1:172.21.15.116:6835/1570267385] up:active 2024-09-20T19:34:42.541 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:42 smithi118 ceph-mon[25856]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 2024-09-20T19:34:42.876 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:34:42.876 INFO:teuthology.orchestra.run.smithi116.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T19:28:02.827016Z", "last_refresh": "2024-09-20T19:31:39.952114Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T19:27:56.093943Z", "last_refresh": "2024-09-20T19:31:39.952176Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T19:27:59.549128Z", "last_refresh": "2024-09-20T19:31:39.952235Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T19:34:35.230375Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T19:34:35.225490Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T19:27:54.712090Z", "last_refresh": "2024-09-20T19:31:39.952028Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi116:172.21.15.116=smithi116", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T19:28:45.778166Z", "last_refresh": "2024-09-20T19:31:39.951888Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T19:28:01.506994Z", "last_refresh": "2024-09-20T19:31:39.952293Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T19:31:39.952412Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T19:27:57.573016Z", "last_refresh": "2024-09-20T19:31:39.952352Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T19:34:43.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 149 B/s wr, 0 op/s 2024-09-20T19:34:43.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:43.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:43.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:43.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:43.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:boot 2024-09-20T19:34:43.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:34:43.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:43 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"}]: dispatch 2024-09-20T19:34:43.846 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 149 B/s wr, 0 op/s 2024-09-20T19:34:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:boot 2024-09-20T19:34:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:34:43.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:43 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"}]: dispatch 2024-09-20T19:34:43.960 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-20T19:34:44.813 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:44 smithi118 ceph-mon[25856]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:44.813 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:44 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:44.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:44 smithi116 ceph-mon[23713]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:44.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:44 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:44.961 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T19:34:45.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:45 smithi116 ceph-mon[23713]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-20T19:34:45.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:45 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:45.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:45 smithi118 ceph-mon[25856]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-20T19:34:45.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:45 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:47.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:47 smithi116 ceph-mon[23713]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T19:34:47.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:47 smithi118 ceph-mon[25856]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T19:34:48.904 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:34:48.904 INFO:teuthology.orchestra.run.smithi116.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T19:28:02.827016Z", "last_refresh": "2024-09-20T19:34:48.827248Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T19:27:56.093943Z", "last_refresh": "2024-09-20T19:34:44.561439Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T19:27:59.549128Z", "last_refresh": "2024-09-20T19:34:48.827505Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T19:34:35.230375Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T19:34:35.225490Z", "last_refresh": "2024-09-20T19:34:44.562034Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T19:27:54.712090Z", "last_refresh": "2024-09-20T19:34:44.561582Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi116:172.21.15.116=smithi116", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T19:28:45.778166Z", "last_refresh": "2024-09-20T19:34:44.561652Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T19:28:01.506994Z", "last_refresh": "2024-09-20T19:34:44.561715Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T19:34:44.561776Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T19:27:57.573016Z", "last_refresh": "2024-09-20T19:34:48.827780Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T19:34:49.716 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-20T19:34:49.717 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T19:34:49.726 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:34:49.726 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-20T19:34:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-20T19:34:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:49.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:49.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:49 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-20T19:34:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:49 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:50.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:50 smithi116 ceph-mon[23713]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:50.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:50 smithi118 ceph-mon[25856]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:34:51.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:51 smithi116 ceph-mon[23713]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T19:34:51.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:51 smithi118 ceph-mon[25856]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T19:34:52.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:52 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-20T19:34:52.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:52 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-20T19:34:53.620 INFO:teuthology.orchestra.run.smithi116.stdout:NFS Cluster Created Successfully 2024-09-20T19:34:53.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:53 smithi116 ceph-mon[23713]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:53.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:53 smithi116 ceph-mon[23713]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T19:34:53.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-20T19:34:53.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:53 smithi116 ceph-mon[23713]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T19:34:53.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:53 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-20T19:34:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:53 smithi118 ceph-mon[25856]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:53 smithi118 ceph-mon[25856]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T19:34:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-20T19:34:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:53 smithi118 ceph-mon[25856]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T19:34:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:53 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-20T19:34:54.315 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-20T19:34:54.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-20T19:34:54.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T19:34:54.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: Saving service nfs.foo spec with placement count:2 2024-09-20T19:34:54.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:54.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:54.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:54.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:54.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:54.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:54.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla 2024-09-20T19:34:54.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T19:34:54.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T19:34:54.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T19:34:54.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:34:54.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:54.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T19:34:54.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:34:54.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T19:34:54.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T19:34:54.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:54 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:54.882 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-20T19:34:54.882 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T19:34:54.882 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: Saving service nfs.foo spec with placement count:2 2024-09-20T19:34:54.882 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:54.882 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:34:54.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T19:34:54.884 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T19:34:54.884 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:54 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:55.845 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:55 smithi116 ceph-mon[23713]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T19:34:55.845 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:55 smithi116 ceph-mon[23713]: Rados config object exists: conf-nfs.foo 2024-09-20T19:34:55.845 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:55 smithi116 ceph-mon[23713]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla-rgw 2024-09-20T19:34:55.845 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:55 smithi116 ceph-mon[23713]: Deploying daemon nfs.foo.0.0.smithi116.bfcxla on smithi116 2024-09-20T19:34:55.845 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:55 smithi116 ceph-mon[23713]: pgmap v230: 97 pgs: 1 creating+activating, 31 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T19:34:55.845 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:55 smithi116 ceph-mon[23713]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T19:34:55.845 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:55 smithi116 ceph-mon[23713]: mgrmap e20: smithi116.poxcrq(active, since 5m), standbys: smithi118.rmiazv 2024-09-20T19:34:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:55 smithi118 ceph-mon[25856]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T19:34:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:55 smithi118 ceph-mon[25856]: Rados config object exists: conf-nfs.foo 2024-09-20T19:34:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:55 smithi118 ceph-mon[25856]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla-rgw 2024-09-20T19:34:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:55 smithi118 ceph-mon[25856]: Deploying daemon nfs.foo.0.0.smithi116.bfcxla on smithi116 2024-09-20T19:34:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:55 smithi118 ceph-mon[25856]: pgmap v230: 97 pgs: 1 creating+activating, 31 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T19:34:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:55 smithi118 ceph-mon[25856]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T19:34:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:55 smithi118 ceph-mon[25856]: mgrmap e20: smithi116.poxcrq(active, since 5m), standbys: smithi118.rmiazv 2024-09-20T19:34:57.598 INFO:teuthology.orchestra.run.smithi116.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-20T19:34:57.598 INFO:teuthology.orchestra.run.smithi116.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-20T19:34:57.598 INFO:teuthology.orchestra.run.smithi116.stderr:Error EINVAL: invalid command 2024-09-20T19:34:57.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:57 smithi116 ceph-mon[23713]: pgmap v232: 97 pgs: 1 creating+activating, 19 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T19:34:57.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:57.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T19:34:57.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T19:34:57.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T19:34:57.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:34:57.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:57.930 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:34:57.930 INFO:teuthology.orchestra.run.smithi116.stdout: "bind": "/fake", 2024-09-20T19:34:57.931 INFO:teuthology.orchestra.run.smithi116.stdout: "fs": "foofs", 2024-09-20T19:34:57.931 INFO:teuthology.orchestra.run.smithi116.stdout: "path": "/", 2024-09-20T19:34:57.931 INFO:teuthology.orchestra.run.smithi116.stdout: "cluster": "foo", 2024-09-20T19:34:57.931 INFO:teuthology.orchestra.run.smithi116.stdout: "mode": "RW" 2024-09-20T19:34:57.931 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:34:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:57 smithi118 ceph-mon[25856]: pgmap v232: 97 pgs: 1 creating+activating, 19 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T19:34:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:34:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T19:34:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T19:34:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T19:34:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:34:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:34:58.643 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-20T19:34:58.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:58 smithi116 ceph-mon[23713]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr 2024-09-20T19:34:58.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:58 smithi116 ceph-mon[23713]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T19:34:58.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:58 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-20T19:34:58.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:58 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-20T19:34:58.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:58 smithi118 ceph-mon[25856]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr 2024-09-20T19:34:58.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:58 smithi118 ceph-mon[25856]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T19:34:58.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:58 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-20T19:34:58.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:58 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-20T19:34:59.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:59 smithi116 ceph-mon[23713]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:59.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:34:59 smithi116 ceph-mon[23713]: pgmap v233: 97 pgs: 1 creating+activating, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T19:34:59.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:59 smithi118 ceph-mon[25856]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:34:59.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:34:59 smithi118 ceph-mon[25856]: pgmap v233: 97 pgs: 1 creating+activating, 96 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-20T19:35:00.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:00 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T19:35:00.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:00 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:35:00.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:00 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T19:35:00.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:00 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T19:35:00.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:00 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:00.940 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:00 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T19:35:00.940 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:00 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:35:00.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:00 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T19:35:00.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:00 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T19:35:00.941 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:00 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:01.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:01 smithi118 ceph-mon[25856]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-20T19:35:01.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:01 smithi118 ceph-mon[25856]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:01.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:01 smithi118 ceph-mon[25856]: Rados config object exists: conf-nfs.foo 2024-09-20T19:35:01.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:01 smithi118 ceph-mon[25856]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr-rgw 2024-09-20T19:35:01.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:01 smithi118 ceph-mon[25856]: Deploying daemon nfs.foo.1.0.smithi118.xrkvdr on smithi118 2024-09-20T19:35:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:01 smithi116 ceph-mon[23713]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-20T19:35:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:01 smithi116 ceph-mon[23713]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:01 smithi116 ceph-mon[23713]: Rados config object exists: conf-nfs.foo 2024-09-20T19:35:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:01 smithi116 ceph-mon[23713]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr-rgw 2024-09-20T19:35:01.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:01 smithi116 ceph-mon[23713]: Deploying daemon nfs.foo.1.0.smithi118.xrkvdr on smithi118 2024-09-20T19:35:03.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:03 smithi116 ceph-mon[23713]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:03 smithi116 ceph-mon[23713]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-20T19:35:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:03 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:03 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:35:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:03 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:03 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:35:03.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:03 smithi118 ceph-mon[25856]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:03.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:03 smithi118 ceph-mon[25856]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-20T19:35:03.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:03 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:03.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:03 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:35:03.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:03 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:03.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:03 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:35:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T19:35:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-09-20T19:35:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T19:35:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-09-20T19:35:04.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:35:04.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:35:04.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]: dispatch 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:35:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T19:35:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [7, 4]}]': finished 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T19:35:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:06.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:06 smithi118 ceph-mon[25856]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T19:35:07.122 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:06 smithi116 ceph-mon[23713]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T19:35:07.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:07 smithi118 ceph-mon[25856]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:07.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:07 smithi118 ceph-mon[25856]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-20T19:35:07.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:07 smithi116 ceph-mon[23713]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:07.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:07 smithi116 ceph-mon[23713]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-20T19:35:08.338 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-20T19:35:08.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:08.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:08.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:08.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:08.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:35:08.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:35:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:08 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:09.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:09.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:09.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:09.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:09.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:35:09.090 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:09.090 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:35:09.090 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:09.090 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:08 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:09.115 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T19:35:09.124 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:35:09.124 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-20T19:35:09.152 INFO:teuthology.orchestra.run.smithi116.stderr:+ mkdir /mnt/foo 2024-09-20T19:35:09.159 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-20T19:35:09.225 INFO:teuthology.orchestra.run.smithi116.stderr:++ hostname 2024-09-20T19:35:09.225 INFO:teuthology.orchestra.run.smithi116.stderr:+ mount -t nfs smithi116:/fake /mnt/foo -o sync 2024-09-20T19:35:09.654 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-20T19:35:09.718 INFO:teuthology.orchestra.run.smithi116.stderr:+ echo test 2024-09-20T19:35:09.728 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-20T19:35:09.790 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:09 smithi116 ceph-mon[23713]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-20T19:35:09.790 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:09 smithi116 ceph-mon[23713]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:09.792 INFO:teuthology.orchestra.run.smithi116.stderr:+ sync 2024-09-20T19:35:09.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:09 smithi118 ceph-mon[25856]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-20T19:35:09.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:09 smithi118 ceph-mon[25856]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:10.040 INFO:teuthology.run_tasks:Running task parallel... 2024-09-20T19:35:10.050 INFO:teuthology.task.parallel:starting parallel... 2024-09-20T19:35:10.050 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-20T19:35:10.051 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-20T19:35:10.051 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:35:10.052 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-20T19:35:10.052 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-20T19:35:10.052 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-20T19:35:10.053 INFO:teuthology.task.exec:Executing custom commands... 2024-09-20T19:35:10.053 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:35:10.053 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-20T19:35:10.149 INFO:teuthology.orchestra.run.smithi116.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-20T19:35:10.149 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:35:10.149 INFO:teuthology.orchestra.run.smithi116.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-20T19:35:10.150 INFO:teuthology.orchestra.run.smithi116.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-20T19:35:10.250 INFO:teuthology.orchestra.run.smithi116.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-20T19:35:10.250 INFO:teuthology.orchestra.run.smithi116.stdout:releasing clients 2024-09-20T19:35:11.250 INFO:teuthology.orchestra.run.smithi116.stdout: 5 132 29.08 MB/sec warmup 1 sec latency 38.421 ms 2024-09-20T19:35:11.696 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:11 smithi116 ceph-mon[23713]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:11 smithi118 ceph-mon[25856]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 KiB/s wr, 6 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:12.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 301 27.74 MB/sec warmup 2 sec latency 32.074 ms 2024-09-20T19:35:13.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 521 30.20 MB/sec warmup 3 sec latency 29.738 ms 2024-09-20T19:35:13.795 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:13 smithi116 ceph-mon[23713]: pgmap v242: 97 pgs: 97 active+clean; 3.6 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 388 KiB/s wr, 13 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:13.810 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-20T19:35:13.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:13 smithi118 ceph-mon[25856]: pgmap v242: 97 pgs: 97 active+clean; 3.6 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 388 KiB/s wr, 13 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:14.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 810 26.84 MB/sec warmup 4 sec latency 23.935 ms 2024-09-20T19:35:15.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 1211 23.12 MB/sec warmup 5 sec latency 47.007 ms 2024-09-20T19:35:15.820 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:15 smithi116 ceph-mon[23713]: pgmap v243: 97 pgs: 97 active+clean; 36 MiB data, 63 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s wr, 73 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:15.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:15 smithi118 ceph-mon[25856]: pgmap v243: 97 pgs: 97 active+clean; 36 MiB data, 63 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s wr, 73 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:16.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 2312 23.00 MB/sec warmup 6 sec latency 21.734 ms 2024-09-20T19:35:17.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 3063 24.15 MB/sec warmup 7 sec latency 32.547 ms 2024-09-20T19:35:17.860 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:17 smithi116 ceph-mon[23713]: pgmap v244: 97 pgs: 97 active+clean; 85 MiB data, 147 MiB used, 715 GiB / 715 GiB avail; 196 B/s rd, 8.3 MiB/s wr, 389 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:17.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:17 smithi118 ceph-mon[25856]: pgmap v244: 97 pgs: 97 active+clean; 85 MiB data, 147 MiB used, 715 GiB / 715 GiB avail; 196 B/s rd, 8.3 MiB/s wr, 389 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:18.187 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-20T19:35:18.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 3677 24.24 MB/sec warmup 8 sec latency 29.142 ms 2024-09-20T19:35:19.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 3891 22.16 MB/sec warmup 9 sec latency 123.923 ms 2024-09-20T19:35:19.706 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:19 smithi116 ceph-mon[23713]: pgmap v245: 97 pgs: 97 active+clean; 111 MiB data, 171 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 9.8 MiB/s wr, 411 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:19.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:19 smithi118 ceph-mon[25856]: pgmap v245: 97 pgs: 97 active+clean; 111 MiB data, 171 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 9.8 MiB/s wr, 411 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:20.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 4178 20.11 MB/sec warmup 10 sec latency 83.973 ms 2024-09-20T19:35:21.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 4673 19.20 MB/sec warmup 11 sec latency 44.423 ms 2024-09-20T19:35:21.718 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:21 smithi116 ceph-mon[23713]: pgmap v246: 97 pgs: 97 active+clean; 124 MiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 12 MiB/s wr, 635 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:21.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:21 smithi118 ceph-mon[25856]: pgmap v246: 97 pgs: 97 active+clean; 124 MiB data, 209 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 12 MiB/s wr, 635 op/s; 0 B/s, 0 objects/s recovering 2024-09-20T19:35:22.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 5421 18.99 MB/sec warmup 12 sec latency 45.237 ms 2024-09-20T19:35:23.221 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-20T19:35:23.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 5905 18.03 MB/sec warmup 13 sec latency 75.616 ms 2024-09-20T19:35:23.699 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:23 smithi116 ceph-mon[23713]: pgmap v247: 97 pgs: 97 active+clean; 126 MiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 13 MiB/s wr, 648 op/s 2024-09-20T19:35:23.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:23 smithi118 ceph-mon[25856]: pgmap v247: 97 pgs: 97 active+clean; 126 MiB data, 227 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 13 MiB/s wr, 648 op/s 2024-09-20T19:35:24.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 6748 19.38 MB/sec warmup 14 sec latency 59.759 ms 2024-09-20T19:35:24.757 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:24 smithi116 ceph-mon[23713]: pgmap v248: 97 pgs: 97 active+clean; 128 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 14 MiB/s wr, 762 op/s 2024-09-20T19:35:25.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:24 smithi118 ceph-mon[25856]: pgmap v248: 97 pgs: 97 active+clean; 128 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 14 MiB/s wr, 762 op/s 2024-09-20T19:35:25.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 7382 19.64 MB/sec warmup 15 sec latency 51.152 ms 2024-09-20T19:35:26.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 7607 18.53 MB/sec warmup 16 sec latency 76.528 ms 2024-09-20T19:35:27.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 7966 17.65 MB/sec warmup 17 sec latency 89.782 ms 2024-09-20T19:35:27.316 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:27 smithi116 ceph-mon[23713]: pgmap v249: 97 pgs: 97 active+clean; 136 MiB data, 289 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 12 MiB/s wr, 924 op/s 2024-09-20T19:35:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:27 smithi118 ceph-mon[25856]: pgmap v249: 97 pgs: 97 active+clean; 136 MiB data, 289 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 12 MiB/s wr, 924 op/s 2024-09-20T19:35:28.245 INFO:teuthology.orchestra.run.smithi116.stderr:module 'nfs' is already enabled 2024-09-20T19:35:28.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 8259 17.09 MB/sec warmup 18 sec latency 59.474 ms 2024-09-20T19:35:28.333 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:28 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1219321870' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-20T19:35:28.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:28 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1219321870' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-20T19:35:29.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 9219 17.13 MB/sec warmup 19 sec latency 30.415 ms 2024-09-20T19:35:29.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:29 smithi116 ceph-mon[23713]: pgmap v250: 97 pgs: 97 active+clean; 137 MiB data, 327 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 9.2 MiB/s wr, 693 op/s 2024-09-20T19:35:29.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:29 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1219321870' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-20T19:35:29.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:29 smithi116 ceph-mon[23713]: mgrmap e21: smithi116.poxcrq(active, since 6m), standbys: smithi118.rmiazv 2024-09-20T19:35:29.710 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-20T19:35:29.713 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:29 smithi118 ceph-mon[25856]: pgmap v250: 97 pgs: 97 active+clean; 137 MiB data, 327 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 9.2 MiB/s wr, 693 op/s 2024-09-20T19:35:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:29 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1219321870' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-20T19:35:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:29 smithi118 ceph-mon[25856]: mgrmap e21: smithi116.poxcrq(active, since 6m), standbys: smithi118.rmiazv 2024-09-20T19:35:30.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 9532 16.60 MB/sec warmup 20 sec latency 65.697 ms 2024-09-20T19:35:31.249 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:31 smithi116 ceph-mon[23713]: pgmap v251: 97 pgs: 97 active+clean; 146 MiB data, 375 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 874 op/s 2024-09-20T19:35:31.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 9881 16.54 MB/sec warmup 21 sec latency 72.473 ms 2024-09-20T19:35:31.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:31 smithi118 ceph-mon[25856]: pgmap v251: 97 pgs: 97 active+clean; 146 MiB data, 375 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 874 op/s 2024-09-20T19:35:32.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 10358 16.67 MB/sec warmup 22 sec latency 42.168 ms 2024-09-20T19:35:32.573 INFO:teuthology.orchestra.run.smithi116.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 2024-09-20T19:35:33.251 INFO:teuthology.orchestra.run.smithi116.stdout: 5 10800 16.81 MB/sec warmup 23 sec latency 43.640 ms 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: pgmap v252: 97 pgs: 97 active+clean; 146 MiB data, 381 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 664 op/s 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:35:33.770 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:33.771 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:33 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:33.855 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-20T19:35:33.856 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:35:33.856 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: pgmap v252: 97 pgs: 97 active+clean; 146 MiB data, 381 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 664 op/s 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:33.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:33 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:35:34.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 11108 16.32 MB/sec warmup 24 sec latency 37.837 ms 2024-09-20T19:35:34.784 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:34 smithi116 ceph-mon[23713]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 2024-09-20T19:35:34.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:34 smithi118 ceph-mon[25856]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 2024-09-20T19:35:35.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 11412 15.78 MB/sec warmup 25 sec latency 68.795 ms 2024-09-20T19:35:35.793 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:35 smithi116 ceph-mon[23713]: pgmap v253: 97 pgs: 97 active+clean; 150 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 812 op/s 2024-09-20T19:35:35.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:35 smithi118 ceph-mon[25856]: pgmap v253: 97 pgs: 97 active+clean; 150 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 812 op/s 2024-09-20T19:35:36.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 11781 15.48 MB/sec warmup 26 sec latency 33.275 ms 2024-09-20T19:35:37.059 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:35:37.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 12157 15.24 MB/sec warmup 27 sec latency 55.048 ms 2024-09-20T19:35:37.816 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:37 smithi116 ceph-mon[23713]: pgmap v254: 97 pgs: 97 active+clean; 153 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.5 MiB/s wr, 746 op/s 2024-09-20T19:35:37.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:37 smithi118 ceph-mon[25856]: pgmap v254: 97 pgs: 97 active+clean; 153 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.5 MiB/s wr, 746 op/s 2024-09-20T19:35:38.124 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:35:38.124 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (5m) 30s ago 6m 22.8M - ba2b418f427c 2e1af9a638b9 2024-09-20T19:35:38.124 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (6m) 30s ago 6m 6413k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (6m) 32s ago 6m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (5m) 30s ago 6m 54.2M - 8.3.5 dad864ee21e9 564eedfb2837 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (59s) 30s ago 57s 12.4M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (56s) 32s ago 55s 12.1M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283 running (8m) 30s ago 8m 459M - 17.2.0 e1d6a67b021e 401b463eca11 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 running (6m) 32s ago 6m 411M - 17.2.0 e1d6a67b021e 6154e3367da1 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (8m) 30s ago 8m 47.4M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (6m) 32s ago 6m 36.2M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (41s) 30s ago 40s 46.9M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (35s) 32s ago 35s 50.7M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (6m) 30s ago 6m 17.3M - 1dbe0e931976 b18146bfd761 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 32s ago 6m 17.4M - 1dbe0e931976 2e0ae7aea427 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (5m) 30s ago 5m 45.3M 3216M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:35:38.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (4m) 30s ago 4m 45.0M 3216M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:35:38.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (4m) 30s ago 4m 39.8M 3216M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:35:38.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (4m) 30s ago 4m 40.9M 3216M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:35:38.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (3m) 32s ago 3m 40.2M 3984M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:35:38.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (3m) 32s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:35:38.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (2m) 32s ago 2m 38.9M 3984M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:35:38.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (2m) 32s ago 2m 39.7M 3984M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:35:38.126 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (5m) 30s ago 5m 60.4M - 514e6a882f6e b631805c5975 2024-09-20T19:35:38.253 INFO:teuthology.orchestra.run.smithi116.stdout: 5 12612 15.01 MB/sec warmup 28 sec latency 65.974 ms 2024-09-20T19:35:38.733 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:35:38.734 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:35:38.735 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:35:38.735 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T19:35:38.735 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:35:38.735 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:35:38.825 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:38 smithi116 ceph-mon[23713]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:38.825 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:38 smithi116 ceph-mon[23713]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:38.936 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:38 smithi118 ceph-mon[25856]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:38.936 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:38 smithi118 ceph-mon[25856]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:39.159 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:35:39.159 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7", 2024-09-20T19:35:39.160 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:35:39.160 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [], 2024-09-20T19:35:39.160 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "", 2024-09-20T19:35:39.160 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 image" 2024-09-20T19:35:39.160 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:35:39.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 12933 14.63 MB/sec warmup 29 sec latency 48.468 ms 2024-09-20T19:35:39.773 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_OK 2024-09-20T19:35:39.830 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:39 smithi116 ceph-mon[23713]: pgmap v255: 97 pgs: 97 active+clean; 156 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.9 MiB/s wr, 606 op/s 2024-09-20T19:35:39.830 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:39 smithi116 ceph-mon[23713]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:39.830 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:39 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3500830313' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:35:39.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:39 smithi118 ceph-mon[25856]: pgmap v255: 97 pgs: 97 active+clean; 156 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.9 MiB/s wr, 606 op/s 2024-09-20T19:35:39.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:39 smithi118 ceph-mon[25856]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:39.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:39 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3500830313' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:35:40.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 13361 14.65 MB/sec warmup 30 sec latency 50.299 ms 2024-09-20T19:35:40.634 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:40 smithi116 ceph-mon[23713]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:40.634 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:40 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2074072842' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:35:40.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:40 smithi118 ceph-mon[25856]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:35:40.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:40 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2074072842' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:35:41.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 13812 14.87 MB/sec warmup 31 sec latency 84.715 ms 2024-09-20T19:35:41.654 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:41 smithi116 ceph-mon[23713]: pgmap v256: 97 pgs: 97 active+clean; 154 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 724 op/s 2024-09-20T19:35:41.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:41 smithi118 ceph-mon[25856]: pgmap v256: 97 pgs: 97 active+clean; 154 MiB data, 462 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 724 op/s 2024-09-20T19:35:42.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 14355 15.07 MB/sec warmup 32 sec latency 43.272 ms 2024-09-20T19:35:43.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 14816 14.80 MB/sec warmup 33 sec latency 44.603 ms 2024-09-20T19:35:43.684 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:43 smithi116 ceph-mon[23713]: pgmap v257: 97 pgs: 97 active+clean; 153 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.9 MiB/s wr, 479 op/s 2024-09-20T19:35:43.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:43 smithi118 ceph-mon[25856]: pgmap v257: 97 pgs: 97 active+clean; 153 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.9 MiB/s wr, 479 op/s 2024-09-20T19:35:44.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 15238 14.59 MB/sec warmup 34 sec latency 44.396 ms 2024-09-20T19:35:45.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 16030 14.71 MB/sec warmup 35 sec latency 31.011 ms 2024-09-20T19:35:45.726 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:45 smithi116 ceph-mon[23713]: pgmap v258: 97 pgs: 97 active+clean; 161 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-09-20T19:35:45.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:45 smithi118 ceph-mon[25856]: pgmap v258: 97 pgs: 97 active+clean; 161 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.3 MiB/s wr, 626 op/s 2024-09-20T19:35:46.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 16805 14.81 MB/sec warmup 36 sec latency 33.818 ms 2024-09-20T19:35:47.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 17363 15.07 MB/sec warmup 37 sec latency 38.829 ms 2024-09-20T19:35:47.762 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:47 smithi116 ceph-mon[23713]: pgmap v259: 97 pgs: 97 active+clean; 172 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 569 op/s 2024-09-20T19:35:47.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:47 smithi118 ceph-mon[25856]: pgmap v259: 97 pgs: 97 active+clean; 172 MiB data, 538 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 569 op/s 2024-09-20T19:35:48.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 17934 15.21 MB/sec warmup 38 sec latency 43.465 ms 2024-09-20T19:35:49.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 18252 14.95 MB/sec warmup 39 sec latency 32.582 ms 2024-09-20T19:35:49.776 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:49 smithi116 ceph-mon[23713]: pgmap v260: 97 pgs: 97 active+clean; 172 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 625 op/s 2024-09-20T19:35:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:49 smithi118 ceph-mon[25856]: pgmap v260: 97 pgs: 97 active+clean; 172 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 625 op/s 2024-09-20T19:35:50.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 18721 14.77 MB/sec warmup 40 sec latency 33.767 ms 2024-09-20T19:35:51.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 19388 14.66 MB/sec warmup 41 sec latency 36.262 ms 2024-09-20T19:35:51.796 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:51 smithi116 ceph-mon[23713]: pgmap v261: 97 pgs: 97 active+clean; 178 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 760 op/s 2024-09-20T19:35:51.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:51 smithi118 ceph-mon[25856]: pgmap v261: 97 pgs: 97 active+clean; 178 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 760 op/s 2024-09-20T19:35:52.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 20057 14.73 MB/sec warmup 42 sec latency 29.594 ms 2024-09-20T19:35:53.254 INFO:teuthology.orchestra.run.smithi116.stdout: 5 20484 14.67 MB/sec warmup 43 sec latency 60.623 ms 2024-09-20T19:35:53.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:53 smithi116 ceph-mon[23713]: pgmap v262: 97 pgs: 97 active+clean; 179 MiB data, 632 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.0 MiB/s wr, 607 op/s 2024-09-20T19:35:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:53 smithi118 ceph-mon[25856]: pgmap v262: 97 pgs: 97 active+clean; 179 MiB data, 632 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.0 MiB/s wr, 607 op/s 2024-09-20T19:35:54.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 20977 14.84 MB/sec warmup 44 sec latency 45.219 ms 2024-09-20T19:35:55.252 INFO:teuthology.orchestra.run.smithi116.stdout: 5 21271 14.72 MB/sec warmup 45 sec latency 69.461 ms 2024-09-20T19:35:55.835 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:55 smithi116 ceph-mon[23713]: pgmap v263: 97 pgs: 97 active+clean; 181 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 731 op/s 2024-09-20T19:35:55.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:55 smithi118 ceph-mon[25856]: pgmap v263: 97 pgs: 97 active+clean; 181 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 731 op/s 2024-09-20T19:35:56.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 21733 14.72 MB/sec warmup 46 sec latency 42.386 ms 2024-09-20T19:35:57.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 22052 14.52 MB/sec warmup 47 sec latency 41.376 ms 2024-09-20T19:35:57.862 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:57 smithi116 ceph-mon[23713]: pgmap v264: 97 pgs: 97 active+clean; 188 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 741 op/s 2024-09-20T19:35:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:57 smithi118 ceph-mon[25856]: pgmap v264: 97 pgs: 97 active+clean; 188 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 741 op/s 2024-09-20T19:35:58.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 22420 14.38 MB/sec warmup 48 sec latency 72.288 ms 2024-09-20T19:35:59.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 22719 14.16 MB/sec warmup 49 sec latency 72.817 ms 2024-09-20T19:35:59.900 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:35:59 smithi116 ceph-mon[23713]: pgmap v265: 97 pgs: 97 active+clean; 189 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 673 op/s 2024-09-20T19:35:59.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:35:59 smithi118 ceph-mon[25856]: pgmap v265: 97 pgs: 97 active+clean; 189 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 673 op/s 2024-09-20T19:36:00.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 23065 14.03 MB/sec warmup 50 sec latency 66.839 ms 2024-09-20T19:36:01.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 23355 13.91 MB/sec warmup 51 sec latency 59.727 ms 2024-09-20T19:36:01.688 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:01 smithi116 ceph-mon[23713]: pgmap v266: 97 pgs: 97 active+clean; 194 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 764 op/s 2024-09-20T19:36:01.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:01 smithi118 ceph-mon[25856]: pgmap v266: 97 pgs: 97 active+clean; 194 MiB data, 743 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 764 op/s 2024-09-20T19:36:02.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 23834 13.86 MB/sec warmup 52 sec latency 58.069 ms 2024-09-20T19:36:03.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 24127 13.77 MB/sec warmup 53 sec latency 46.937 ms 2024-09-20T19:36:03.794 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:03 smithi116 ceph-mon[23713]: pgmap v267: 97 pgs: 97 active+clean; 194 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 586 op/s 2024-09-20T19:36:03.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:03 smithi118 ceph-mon[25856]: pgmap v267: 97 pgs: 97 active+clean; 194 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 586 op/s 2024-09-20T19:36:04.262 INFO:teuthology.orchestra.run.smithi116.stdout: 5 24550 13.91 MB/sec warmup 54 sec latency 59.587 ms 2024-09-20T19:36:04.842 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:36:04.842 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:36:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:36:04.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:36:05.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 25008 13.94 MB/sec warmup 55 sec latency 57.205 ms 2024-09-20T19:36:05.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:05 smithi118 ceph-mon[25856]: pgmap v268: 97 pgs: 97 active+clean; 191 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 653 op/s 2024-09-20T19:36:05.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:05.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:36:05.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:05.866 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:05 smithi116 ceph-mon[23713]: pgmap v268: 97 pgs: 97 active+clean; 191 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 653 op/s 2024-09-20T19:36:05.866 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:05.867 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:36:05.867 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:06.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 25361 13.82 MB/sec warmup 56 sec latency 43.237 ms 2024-09-20T19:36:06.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:06 smithi116 ceph-mon[23713]: Upgrade: Target is version 19.3.0-5127-ge8baca1e (unknown) 2024-09-20T19:36:06.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:06 smithi116 ceph-mon[23713]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d'] 2024-09-20T19:36:06.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:06 smithi116 ceph-mon[23713]: Upgrade: Need to upgrade myself (mgr.smithi116.poxcrq) 2024-09-20T19:36:06.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:06 smithi118 ceph-mon[25856]: Upgrade: Target is version 19.3.0-5127-ge8baca1e (unknown) 2024-09-20T19:36:06.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:06 smithi118 ceph-mon[25856]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d'] 2024-09-20T19:36:06.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:06 smithi118 ceph-mon[25856]: Upgrade: Need to upgrade myself (mgr.smithi116.poxcrq) 2024-09-20T19:36:07.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 25830 13.68 MB/sec warmup 57 sec latency 65.216 ms 2024-09-20T19:36:07.888 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:07 smithi116 ceph-mon[23713]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d on smithi118 2024-09-20T19:36:07.888 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:07 smithi116 ceph-mon[23713]: pgmap v269: 97 pgs: 97 active+clean; 192 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 548 op/s 2024-09-20T19:36:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:07 smithi118 ceph-mon[25856]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d on smithi118 2024-09-20T19:36:07.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:07 smithi118 ceph-mon[25856]: pgmap v269: 97 pgs: 97 active+clean; 192 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 548 op/s 2024-09-20T19:36:08.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 26428 13.70 MB/sec warmup 58 sec latency 65.164 ms 2024-09-20T19:36:09.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 27067 13.66 MB/sec warmup 59 sec latency 71.160 ms 2024-09-20T19:36:09.704 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:09 smithi116 ceph-mon[23713]: pgmap v270: 97 pgs: 97 active+clean; 204 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 559 op/s 2024-09-20T19:36:09.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:09 smithi118 ceph-mon[25856]: pgmap v270: 97 pgs: 97 active+clean; 204 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 559 op/s 2024-09-20T19:36:10.184 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:36:10.261 INFO:teuthology.orchestra.run.smithi116.stdout: 5 27662 13.73 MB/sec warmup 60 sec latency 43.288 ms 2024-09-20T19:36:11.110 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:36:11.110 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (6m) 63s ago 7m 22.8M - ba2b418f427c 2e1af9a638b9 2024-09-20T19:36:11.110 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (7m) 63s ago 7m 6413k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:36:11.110 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (6m) 65s ago 6m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:36:11.110 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (5m) 63s ago 7m 54.2M - 8.3.5 dad864ee21e9 564eedfb2837 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (92s) 63s ago 90s 12.4M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (89s) 65s ago 88s 12.1M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283 running (8m) 63s ago 8m 459M - 17.2.0 e1d6a67b021e 401b463eca11 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 running (6m) 65s ago 6m 411M - 17.2.0 e1d6a67b021e 6154e3367da1 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (8m) 63s ago 8m 47.4M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (6m) 65s ago 6m 36.2M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (74s) 63s ago 73s 46.9M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (68s) 65s ago 68s 50.7M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (7m) 63s ago 7m 17.3M - 1dbe0e931976 b18146bfd761 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 65s ago 6m 17.4M - 1dbe0e931976 2e0ae7aea427 2024-09-20T19:36:11.111 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (6m) 63s ago 6m 45.3M 3216M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (5m) 63s ago 5m 45.0M 3216M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (5m) 63s ago 5m 39.8M 3216M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (4m) 63s ago 4m 40.9M 3216M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (4m) 65s ago 4m 40.2M 3984M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (3m) 65s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (3m) 65s ago 3m 38.9M 3984M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (3m) 65s ago 3m 39.7M 3984M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:36:11.112 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (6m) 63s ago 6m 60.4M - 514e6a882f6e b631805c5975 2024-09-20T19:36:11.263 INFO:teuthology.orchestra.run.smithi116.stdout: 5 28401 14.02 MB/sec warmup 61 sec latency 71.910 ms 2024-09-20T19:36:11.717 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:36:11.717 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:36:11.717 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:36:11.717 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:11.717 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:36:11.718 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:36:11.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:11 smithi116 ceph-mon[23713]: pgmap v271: 97 pgs: 97 active+clean; 210 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 686 op/s 2024-09-20T19:36:11.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:11 smithi116 ceph-mon[23713]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:11.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:11 smithi116 ceph-mon[23713]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:11 smithi118 ceph-mon[25856]: pgmap v271: 97 pgs: 97 active+clean; 210 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 686 op/s 2024-09-20T19:36:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:11 smithi118 ceph-mon[25856]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:11.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:11 smithi118 ceph-mon[25856]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:12.263 INFO:teuthology.orchestra.run.smithi116.stdout: 5 29030 14.05 MB/sec warmup 62 sec latency 18.726 ms 2024-09-20T19:36:12.264 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:36:12.264 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:36:12.264 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:36:12.264 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [], 2024-09-20T19:36:12.264 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "0/23 daemons upgraded", 2024-09-20T19:36:12.265 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d image on host smithi118" 2024-09-20T19:36:12.265 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:36:12.726 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:12 smithi116 ceph-mon[23713]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:12.726 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:12 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3417689997' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:36:12.890 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_OK 2024-09-20T19:36:12.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:12 smithi118 ceph-mon[25856]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:12.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:12 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3417689997' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:36:13.263 INFO:teuthology.orchestra.run.smithi116.stdout: 5 29772 14.06 MB/sec warmup 63 sec latency 15.031 ms 2024-09-20T19:36:13.730 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:13 smithi116 ceph-mon[23713]: pgmap v272: 97 pgs: 97 active+clean; 209 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-09-20T19:36:13.730 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:13 smithi116 ceph-mon[23713]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:13.730 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:13 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3177082764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:36:13.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:13 smithi118 ceph-mon[25856]: pgmap v272: 97 pgs: 97 active+clean; 209 MiB data, 874 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 505 op/s 2024-09-20T19:36:13.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:13 smithi118 ceph-mon[25856]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:13.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:13 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3177082764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:36:14.263 INFO:teuthology.orchestra.run.smithi116.stdout: 5 30837 14.22 MB/sec warmup 64 sec latency 56.294 ms 2024-09-20T19:36:14.735 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:14 smithi116 ceph-mon[23713]: pgmap v273: 97 pgs: 97 active+clean; 215 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 622 op/s 2024-09-20T19:36:14.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:14 smithi118 ceph-mon[25856]: pgmap v273: 97 pgs: 97 active+clean; 215 MiB data, 882 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 622 op/s 2024-09-20T19:36:15.263 INFO:teuthology.orchestra.run.smithi116.stdout: 5 31877 14.59 MB/sec warmup 65 sec latency 20.538 ms 2024-09-20T19:36:16.263 INFO:teuthology.orchestra.run.smithi116.stdout: 5 32705 14.68 MB/sec warmup 66 sec latency 20.842 ms 2024-09-20T19:36:17.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 33556 14.71 MB/sec warmup 67 sec latency 32.394 ms 2024-09-20T19:36:17.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:17 smithi116 ceph-mon[23713]: pgmap v274: 97 pgs: 97 active+clean; 227 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.2 MiB/s wr, 801 op/s 2024-09-20T19:36:17.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:17 smithi118 ceph-mon[25856]: pgmap v274: 97 pgs: 97 active+clean; 227 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.2 MiB/s wr, 801 op/s 2024-09-20T19:36:18.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 34414 14.81 MB/sec warmup 68 sec latency 23.150 ms 2024-09-20T19:36:19.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 35213 15.03 MB/sec warmup 69 sec latency 28.800 ms 2024-09-20T19:36:19.406 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:19 smithi116 ceph-mon[23713]: pgmap v275: 97 pgs: 97 active+clean; 228 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.2 MiB/s wr, 849 op/s 2024-09-20T19:36:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:19 smithi118 ceph-mon[25856]: pgmap v275: 97 pgs: 97 active+clean; 228 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.2 MiB/s wr, 849 op/s 2024-09-20T19:36:20.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 35905 15.12 MB/sec warmup 70 sec latency 23.169 ms 2024-09-20T19:36:21.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 36576 15.10 MB/sec warmup 71 sec latency 17.837 ms 2024-09-20T19:36:21.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:21 smithi116 ceph-mon[23713]: pgmap v276: 97 pgs: 97 active+clean; 239 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-09-20T19:36:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:21 smithi118 ceph-mon[25856]: pgmap v276: 97 pgs: 97 active+clean; 239 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-09-20T19:36:22.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 37478 15.15 MB/sec warmup 72 sec latency 16.845 ms 2024-09-20T19:36:23.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 38382 15.28 MB/sec warmup 73 sec latency 21.671 ms 2024-09-20T19:36:23.430 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:23 smithi116 ceph-mon[23713]: pgmap v277: 97 pgs: 97 active+clean; 239 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 941 op/s 2024-09-20T19:36:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:23 smithi118 ceph-mon[25856]: pgmap v277: 97 pgs: 97 active+clean; 239 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 941 op/s 2024-09-20T19:36:24.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 39120 15.48 MB/sec warmup 74 sec latency 21.895 ms 2024-09-20T19:36:25.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 39804 15.48 MB/sec warmup 75 sec latency 27.187 ms 2024-09-20T19:36:25.442 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:25 smithi116 ceph-mon[23713]: pgmap v278: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.18k op/s 2024-09-20T19:36:25.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:25 smithi118 ceph-mon[25856]: pgmap v278: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 12 MiB/s wr, 1.18k op/s 2024-09-20T19:36:26.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 40700 15.49 MB/sec warmup 76 sec latency 60.851 ms 2024-09-20T19:36:27.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:27 smithi116 ceph-mon[23713]: pgmap v279: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-09-20T19:36:27.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 41763 15.65 MB/sec warmup 77 sec latency 20.743 ms 2024-09-20T19:36:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:27 smithi118 ceph-mon[25856]: pgmap v279: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.11k op/s 2024-09-20T19:36:28.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 42606 15.88 MB/sec warmup 78 sec latency 17.707 ms 2024-09-20T19:36:29.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 43005 15.82 MB/sec warmup 79 sec latency 60.193 ms 2024-09-20T19:36:29.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:29 smithi116 ceph-mon[23713]: pgmap v280: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.02k op/s 2024-09-20T19:36:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:29 smithi118 ceph-mon[25856]: pgmap v280: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.02k op/s 2024-09-20T19:36:30.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 43216 15.67 MB/sec warmup 80 sec latency 88.961 ms 2024-09-20T19:36:31.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 43472 15.55 MB/sec warmup 81 sec latency 41.592 ms 2024-09-20T19:36:31.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:31 smithi116 ceph-mon[23713]: pgmap v281: 97 pgs: 97 active+clean; 265 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.25k op/s 2024-09-20T19:36:31.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:31 smithi118 ceph-mon[25856]: pgmap v281: 97 pgs: 97 active+clean; 265 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.25k op/s 2024-09-20T19:36:32.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 43863 15.44 MB/sec warmup 82 sec latency 53.785 ms 2024-09-20T19:36:33.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 44271 15.36 MB/sec warmup 83 sec latency 40.849 ms 2024-09-20T19:36:33.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:33 smithi116 ceph-mon[23713]: pgmap v282: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 845 op/s 2024-09-20T19:36:33.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:33 smithi118 ceph-mon[25856]: pgmap v282: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 845 op/s 2024-09-20T19:36:34.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 44771 15.33 MB/sec warmup 84 sec latency 32.649 ms 2024-09-20T19:36:35.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 45698 15.48 MB/sec warmup 85 sec latency 22.574 ms 2024-09-20T19:36:35.456 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:35 smithi116 ceph-mon[23713]: pgmap v283: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 955 op/s 2024-09-20T19:36:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:35 smithi118 ceph-mon[25856]: pgmap v283: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 955 op/s 2024-09-20T19:36:36.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 46053 15.49 MB/sec warmup 86 sec latency 71.750 ms 2024-09-20T19:36:37.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:37 smithi116 ceph-mon[23713]: pgmap v284: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 795 op/s 2024-09-20T19:36:37.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:37 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:37.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:37 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:36:37.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:37 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:36:37.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:37 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:36:37.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 46414 15.39 MB/sec warmup 87 sec latency 53.568 ms 2024-09-20T19:36:37.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:37 smithi118 ceph-mon[25856]: pgmap v284: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 795 op/s 2024-09-20T19:36:37.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:37 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:37.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:37 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:36:37.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:37 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:36:37.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:37 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:36:38.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:38 smithi116 ceph-mon[23713]: Upgrade: Updating mgr.smithi118.rmiazv 2024-09-20T19:36:38.245 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:38 smithi116 ceph-mon[23713]: Deploying daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:36:38.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 46684 15.31 MB/sec warmup 88 sec latency 77.952 ms 2024-09-20T19:36:38.461 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:38 smithi118 ceph-mon[25856]: Upgrade: Updating mgr.smithi118.rmiazv 2024-09-20T19:36:38.461 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:38 smithi118 ceph-mon[25856]: Deploying daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:36:39.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 46991 15.23 MB/sec warmup 89 sec latency 83.951 ms 2024-09-20T19:36:39.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:39 smithi116 ceph-mon[23713]: pgmap v285: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 832 op/s 2024-09-20T19:36:39.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:39 smithi118 ceph-mon[25856]: pgmap v285: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 832 op/s 2024-09-20T19:36:40.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 47350 15.12 MB/sec warmup 90 sec latency 63.706 ms 2024-09-20T19:36:41.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 47847 15.05 MB/sec warmup 91 sec latency 87.244 ms 2024-09-20T19:36:41.290 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:41 smithi116 ceph-mon[23713]: pgmap v286: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 790 op/s 2024-09-20T19:36:41.515 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:41 smithi118 ceph-mon[25856]: pgmap v286: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 790 op/s 2024-09-20T19:36:42.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 48449 15.10 MB/sec warmup 92 sec latency 35.811 ms 2024-09-20T19:36:42.556 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:42 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:42.556 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:42 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:36:42.556 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:42 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:36:42.556 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:42 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:36:42.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:42 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:42.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:42 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:36:42.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:42 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:36:42.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:42 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:36:43.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 49024 15.10 MB/sec warmup 93 sec latency 48.064 ms 2024-09-20T19:36:43.412 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:36:43.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:43 smithi116 ceph-mon[23713]: pgmap v287: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 493 op/s 2024-09-20T19:36:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:43 smithi118 ceph-mon[25856]: pgmap v287: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 493 op/s 2024-09-20T19:36:44.118 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:36:44.118 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (6m) 96s ago 8m 22.8M - ba2b418f427c 2e1af9a638b9 2024-09-20T19:36:44.118 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (8m) 96s ago 8m 6413k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:36:44.118 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (7m) 98s ago 7m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:36:44.118 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (6m) 96s ago 7m 54.2M - 8.3.5 dad864ee21e9 564eedfb2837 2024-09-20T19:36:44.119 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (2m) 96s ago 2m 12.4M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:36:44.119 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (2m) 98s ago 2m 12.1M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:36:44.119 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283 running (9m) 96s ago 9m 459M - 17.2.0 e1d6a67b021e 401b463eca11 2024-09-20T19:36:44.119 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 starting - - - - 2024-09-20T19:36:44.122 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (9m) 96s ago 9m 47.4M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:36:44.122 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (7m) 98s ago 7m 36.2M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:36:44.123 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (107s) 96s ago 106s 46.9M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:36:44.124 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (101s) 98s ago 101s 50.7M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:36:44.124 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (7m) 96s ago 7m 17.3M - 1dbe0e931976 b18146bfd761 2024-09-20T19:36:44.124 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 98s ago 7m 17.4M - 1dbe0e931976 2e0ae7aea427 2024-09-20T19:36:44.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (6m) 96s ago 6m 45.3M 3216M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:36:44.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (6m) 96s ago 6m 45.0M 3216M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:36:44.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (5m) 96s ago 5m 39.8M 3216M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:36:44.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (5m) 96s ago 5m 40.9M 3216M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:36:44.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (4m) 98s ago 4m 40.2M 3984M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:36:44.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (4m) 98s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:36:44.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (4m) 98s ago 4m 38.9M 3984M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:36:44.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (3m) 98s ago 3m 39.7M 3984M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:36:44.126 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (7m) 96s ago 7m 60.4M - 514e6a882f6e b631805c5975 2024-09-20T19:36:44.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 49285 15.04 MB/sec warmup 94 sec latency 112.530 ms 2024-09-20T19:36:44.554 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:36:44.554 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:36:44.555 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:36:44.594 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:44 smithi116 ceph-mon[23713]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:44 smithi118 ceph-mon[25856]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:44.897 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:36:44.897 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:36:44.897 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:36:44.897 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [], 2024-09-20T19:36:44.897 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "", 2024-09-20T19:36:44.897 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading mgr daemons" 2024-09-20T19:36:44.897 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:36:45.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 49448 14.97 MB/sec warmup 95 sec latency 72.660 ms 2024-09-20T19:36:45.303 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_OK 2024-09-20T19:36:45.637 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:45 smithi116 ceph-mon[23713]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:45.637 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:45 smithi116 ceph-mon[23713]: pgmap v288: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 612 op/s 2024-09-20T19:36:45.637 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:45 smithi116 ceph-mon[23713]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:45.637 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:45 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1833140894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:36:45.637 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:45 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3878405998' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:36:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:45 smithi118 ceph-mon[25856]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:45 smithi118 ceph-mon[25856]: pgmap v288: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 612 op/s 2024-09-20T19:36:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:45 smithi118 ceph-mon[25856]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:45 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1833140894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:36:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:45 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3878405998' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:36:46.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 49610 14.87 MB/sec warmup 96 sec latency 83.907 ms 2024-09-20T19:36:46.711 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:46 smithi116 ceph-mon[23713]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:46.874 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:46 smithi118 ceph-mon[25856]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:36:47.264 INFO:teuthology.orchestra.run.smithi116.stdout: 5 49733 14.75 MB/sec warmup 97 sec latency 139.702 ms 2024-09-20T19:36:47.496 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:47 smithi116 ceph-mon[23713]: pgmap v289: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 619 op/s 2024-09-20T19:36:47.496 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:47 smithi116 ceph-mon[23713]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:36:47.496 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:47 smithi116 ceph-mon[23713]: Standby manager daemon smithi118.rmiazv restarted 2024-09-20T19:36:47.496 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:47 smithi116 ceph-mon[23713]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:36:47.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:47 smithi118 ceph-mon[25856]: pgmap v289: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 619 op/s 2024-09-20T19:36:47.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:47 smithi118 ceph-mon[25856]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:36:47.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:47 smithi118 ceph-mon[25856]: Standby manager daemon smithi118.rmiazv restarted 2024-09-20T19:36:47.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:47 smithi118 ceph-mon[25856]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:36:48.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 50197 14.72 MB/sec warmup 98 sec latency 186.543 ms 2024-09-20T19:36:48.515 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:48 smithi116 ceph-mon[23713]: mgrmap e22: smithi116.poxcrq(active, since 7m), standbys: smithi118.rmiazv 2024-09-20T19:36:48.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:48 smithi118 ceph-mon[25856]: mgrmap e22: smithi116.poxcrq(active, since 7m), standbys: smithi118.rmiazv 2024-09-20T19:36:49.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 50823 14.71 MB/sec warmup 99 sec latency 20.727 ms 2024-09-20T19:36:49.532 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:49 smithi116 ceph-mon[23713]: pgmap v290: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.9 MiB/s wr, 548 op/s 2024-09-20T19:36:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:49 smithi118 ceph-mon[25856]: pgmap v290: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.9 MiB/s wr, 548 op/s 2024-09-20T19:36:50.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 51599 14.68 MB/sec warmup 100 sec latency 30.525 ms 2024-09-20T19:36:51.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 52281 14.74 MB/sec warmup 101 sec latency 51.495 ms 2024-09-20T19:36:51.555 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:51 smithi116 ceph-mon[23713]: pgmap v291: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.3 MiB/s wr, 578 op/s 2024-09-20T19:36:51.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:51 smithi118 ceph-mon[25856]: pgmap v291: 97 pgs: 97 active+clean; 291 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.3 MiB/s wr, 578 op/s 2024-09-20T19:36:52.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 52684 14.76 MB/sec warmup 102 sec latency 113.027 ms 2024-09-20T19:36:53.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 53028 14.74 MB/sec warmup 103 sec latency 60.749 ms 2024-09-20T19:36:53.629 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:53 smithi116 ceph-mon[23713]: pgmap v292: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-20T19:36:53.772 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:53 smithi118 ceph-mon[25856]: pgmap v292: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-20T19:36:54.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 53417 14.69 MB/sec warmup 104 sec latency 128.781 ms 2024-09-20T19:36:55.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 53803 14.68 MB/sec warmup 105 sec latency 241.292 ms 2024-09-20T19:36:55.656 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:55 smithi116 ceph-mon[23713]: pgmap v293: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 635 op/s 2024-09-20T19:36:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:55 smithi118 ceph-mon[25856]: pgmap v293: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 635 op/s 2024-09-20T19:36:56.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 54260 14.62 MB/sec warmup 106 sec latency 31.678 ms 2024-09-20T19:36:57.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 54661 14.52 MB/sec warmup 107 sec latency 81.848 ms 2024-09-20T19:36:57.634 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:57 smithi116 ceph-mon[23713]: pgmap v294: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.2 MiB/s wr, 639 op/s 2024-09-20T19:36:57.634 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:57.634 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:57 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:57.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:57 smithi118 ceph-mon[25856]: pgmap v294: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.2 MiB/s wr, 639 op/s 2024-09-20T19:36:57.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:57.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:57 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:36:58.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 54965 14.47 MB/sec warmup 108 sec latency 73.675 ms 2024-09-20T19:36:59.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 55390 14.48 MB/sec warmup 109 sec latency 50.571 ms 2024-09-20T19:36:59.698 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:36:59 smithi116 ceph-mon[23713]: pgmap v295: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 534 op/s 2024-09-20T19:36:59.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:36:59 smithi118 ceph-mon[25856]: pgmap v295: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 534 op/s 2024-09-20T19:37:00.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 56238 14.55 MB/sec warmup 110 sec latency 29.449 ms 2024-09-20T19:37:01.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 56952 14.62 MB/sec warmup 111 sec latency 16.790 ms 2024-09-20T19:37:01.714 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:01 smithi116 ceph-mon[23713]: pgmap v296: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 719 op/s 2024-09-20T19:37:01.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:01 smithi118 ceph-mon[25856]: pgmap v296: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 719 op/s 2024-09-20T19:37:02.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 57505 14.63 MB/sec warmup 112 sec latency 35.537 ms 2024-09-20T19:37:03.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 57883 14.57 MB/sec warmup 113 sec latency 64.292 ms 2024-09-20T19:37:03.739 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:03 smithi116 ceph-mon[23713]: pgmap v297: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-09-20T19:37:03.922 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:03 smithi118 ceph-mon[25856]: pgmap v297: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-09-20T19:37:04.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 58471 14.54 MB/sec warmup 114 sec latency 66.098 ms 2024-09-20T19:37:05.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:37:05.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:37:05.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:37:05.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:37:05.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: Upgrade: Need to upgrade myself (mgr.smithi116.poxcrq) 2024-09-20T19:37:05.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: Failing over to other MGR 2024-09-20T19:37:05.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr fail", "who": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:37:05.008 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T19:37:05.008 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:37:05.008 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:37:05.008 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:04 smithi116 ceph-mon[23713]: pgmap v299: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 734 op/s 2024-09-20T19:37:05.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: Upgrade: Need to upgrade myself (mgr.smithi116.poxcrq) 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: Failing over to other MGR 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr fail", "who": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:37:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:04 smithi118 ceph-mon[25856]: pgmap v299: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 734 op/s 2024-09-20T19:37:05.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 59092 14.58 MB/sec warmup 115 sec latency 32.535 ms 2024-09-20T19:37:06.019 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "mgr fail", "who": "smithi116.poxcrq"}]': finished 2024-09-20T19:37:06.019 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: mgrmap e23: smithi118.rmiazv(active, starting, since 1.01337s) 2024-09-20T19:37:06.019 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:37:06.019 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:37:06.019 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"}]: dispatch 2024-09-20T19:37:06.019 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"}]: dispatch 2024-09-20T19:37:06.020 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rmiazv", "id": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:37:06.020 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:37:06.020 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:37:06.020 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:37:06.020 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:37:06.020 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:37:06.020 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: Manager daemon smithi118.rmiazv is now available 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.021 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.022 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.022 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.022 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:37:06.022 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:06.022 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:37:06.022 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:37:06.023 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/trash_purge_schedule"}]: dispatch 2024-09-20T19:37:06.023 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:05 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/trash_purge_schedule"}]: dispatch 2024-09-20T19:37:06.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14202 172.21.15.116:0/173833850' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "mgr fail", "who": "smithi116.poxcrq"}]': finished 2024-09-20T19:37:06.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: mgrmap e23: smithi118.rmiazv(active, starting, since 1.01337s) 2024-09-20T19:37:06.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:37:06.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:37:06.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rmiazv", "id": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:37:06.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: Manager daemon smithi118.rmiazv is now available 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:06.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:37:06.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:06.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:37:06.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:37:06.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/trash_purge_schedule"}]: dispatch 2024-09-20T19:37:06.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:05 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi118.rmiazv/trash_purge_schedule"}]: dispatch 2024-09-20T19:37:06.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 59782 14.61 MB/sec warmup 116 sec latency 24.874 ms 2024-09-20T19:37:07.029 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:07 smithi116 ceph-mon[23713]: Checking for cert/key for grafana.smithi116 2024-09-20T19:37:07.029 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:07 smithi116 ceph-mon[23713]: mgrmap e24: smithi118.rmiazv(active, since 2s) 2024-09-20T19:37:07.029 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:07 smithi116 ceph-mon[23713]: pgmap v3: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:07.029 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:07 smithi116 ceph-mon[23713]: Deploying cephadm binary to smithi118 2024-09-20T19:37:07.029 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:07 smithi116 ceph-mon[23713]: Deploying cephadm binary to smithi116 2024-09-20T19:37:07.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 59989 14.54 MB/sec warmup 117 sec latency 54.789 ms 2024-09-20T19:37:07.295 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:07 smithi118 ceph-mon[25856]: Checking for cert/key for grafana.smithi116 2024-09-20T19:37:07.295 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:07 smithi118 ceph-mon[25856]: mgrmap e24: smithi118.rmiazv(active, since 2s) 2024-09-20T19:37:07.295 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:07 smithi118 ceph-mon[25856]: pgmap v3: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:07.296 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:07 smithi118 ceph-mon[25856]: Deploying cephadm binary to smithi118 2024-09-20T19:37:07.296 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:07 smithi118 ceph-mon[25856]: Deploying cephadm binary to smithi116 2024-09-20T19:37:08.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:08 smithi116 ceph-mon[23713]: Health check failed: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) (MGR_MODULE_DEPENDENCY) 2024-09-20T19:37:08.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:08 smithi116 ceph-mon[23713]: pgmap v4: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:08.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:08 smithi116 ceph-mon[23713]: [20/Sep/2024:19:37:07] ENGINE Bus STARTING 2024-09-20T19:37:08.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:08 smithi116 ceph-mon[23713]: [20/Sep/2024:19:37:07] ENGINE Serving on http://172.21.15.118:8765 2024-09-20T19:37:08.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:08 smithi116 ceph-mon[23713]: [20/Sep/2024:19:37:07] ENGINE Serving on https://172.21.15.118:7150 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:08 smithi116 ceph-mon[23713]: [20/Sep/2024:19:37:07] ENGINE Bus STARTED 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:08 smithi116 ceph-mon[23713]: [20/Sep/2024:19:37:07] ENGINE Error in HTTPServer.serve 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Traceback (most recent call last): 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._connections.run(self.expiration_interval) 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._run(expiration_interval) 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: s = self.context.wrap_socket( 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T19:37:08.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: return self.sslsocket_class._create( 2024-09-20T19:37:08.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T19:37:08.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self.do_handshake() 2024-09-20T19:37:08.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T19:37:08.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._sslobj.do_handshake() 2024-09-20T19:37:08.089 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T19:37:08.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:08 smithi118 ceph-mon[25856]: Health check failed: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) (MGR_MODULE_DEPENDENCY) 2024-09-20T19:37:08.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:08 smithi118 ceph-mon[25856]: pgmap v4: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:08.101 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:08 smithi118 ceph-mon[25856]: [20/Sep/2024:19:37:07] ENGINE Bus STARTING 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:08 smithi118 ceph-mon[25856]: [20/Sep/2024:19:37:07] ENGINE Serving on http://172.21.15.118:8765 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:08 smithi118 ceph-mon[25856]: [20/Sep/2024:19:37:07] ENGINE Serving on https://172.21.15.118:7150 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:08 smithi118 ceph-mon[25856]: [20/Sep/2024:19:37:07] ENGINE Bus STARTED 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:08 smithi118 ceph-mon[25856]: [20/Sep/2024:19:37:07] ENGINE Error in HTTPServer.serve 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Traceback (most recent call last): 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._connections.run(self.expiration_interval) 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._run(expiration_interval) 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T19:37:08.102 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s = self.context.wrap_socket( 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: return self.sslsocket_class._create( 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.do_handshake() 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._sslobj.do_handshake() 2024-09-20T19:37:08.103 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T19:37:08.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 60265 14.54 MB/sec warmup 118 sec latency 92.391 ms 2024-09-20T19:37:09.095 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:09 smithi116 ceph-mon[23713]: mgrmap e25: smithi118.rmiazv(active, since 4s) 2024-09-20T19:37:09.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:09 smithi118 ceph-mon[25856]: mgrmap e25: smithi118.rmiazv(active, since 4s) 2024-09-20T19:37:09.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 60627 14.47 MB/sec warmup 119 sec latency 99.014 ms 2024-09-20T19:37:10.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:10 smithi116 ceph-mon[23713]: pgmap v5: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:10.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:10 smithi116 ceph-mon[23713]: mgrmap e26: smithi118.rmiazv(active, since 6s) 2024-09-20T19:37:10.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:10 smithi118 ceph-mon[25856]: pgmap v5: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:10.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:10 smithi118 ceph-mon[25856]: mgrmap e26: smithi118.rmiazv(active, since 6s) 2024-09-20T19:37:11.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 61643 10.13 MB/sec execute 1 sec latency 40.461 ms 2024-09-20T19:37:12.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 62110 10.29 MB/sec execute 2 sec latency 25.443 ms 2024-09-20T19:37:12.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:12 smithi116 ceph-mon[23713]: pgmap v6: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:12.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:12 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:12.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:12 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:12.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:12 smithi118 ceph-mon[25856]: pgmap v6: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-20T19:37:12.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:12 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:12.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:12 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:13.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 62716 13.31 MB/sec execute 3 sec latency 30.133 ms 2024-09-20T19:37:13.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:13 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:13.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:13 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:13.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:13 smithi116 ceph-mon[23713]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.3 MiB/s wr, 399 op/s 2024-09-20T19:37:13.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:13 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:13.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:13 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:13.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:13 smithi118 ceph-mon[25856]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.3 MiB/s wr, 399 op/s 2024-09-20T19:37:14.265 INFO:teuthology.orchestra.run.smithi116.stdout: 5 63345 14.64 MB/sec execute 4 sec latency 29.456 ms 2024-09-20T19:37:14.406 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:14 smithi116 ceph-mon[23713]: Standby manager daemon smithi116.poxcrq started 2024-09-20T19:37:14.406 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:14 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi116.poxcrq/crt"}]: dispatch 2024-09-20T19:37:14.406 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:14 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T19:37:14.407 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:14 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi116.poxcrq/key"}]: dispatch 2024-09-20T19:37:14.407 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:14 smithi116 ceph-mon[23713]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T19:37:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:14 smithi118 ceph-mon[25856]: Standby manager daemon smithi116.poxcrq started 2024-09-20T19:37:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:14 smithi118 ceph-mon[25856]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi116.poxcrq/crt"}]: dispatch 2024-09-20T19:37:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:14 smithi118 ceph-mon[25856]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T19:37:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:14 smithi118 ceph-mon[25856]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi116.poxcrq/key"}]: dispatch 2024-09-20T19:37:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:14 smithi118 ceph-mon[25856]: from='mgr.? 172.21.15.116:0/1998608235' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T19:37:15.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 63823 15.25 MB/sec execute 5 sec latency 115.083 ms 2024-09-20T19:37:15.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:15 smithi116 ceph-mon[23713]: mgrmap e27: smithi118.rmiazv(active, since 10s), standbys: smithi116.poxcrq 2024-09-20T19:37:15.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:15 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:37:15.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:15 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:15.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:15 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:15.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:15 smithi116 ceph-mon[23713]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.3 MiB/s wr, 310 op/s 2024-09-20T19:37:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:15 smithi118 ceph-mon[25856]: mgrmap e27: smithi118.rmiazv(active, since 10s), standbys: smithi116.poxcrq 2024-09-20T19:37:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:15 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:37:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:15 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:15.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:15 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:15.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:15 smithi118 ceph-mon[25856]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.3 MiB/s wr, 310 op/s 2024-09-20T19:37:15.895 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:37:16.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 64283 14.99 MB/sec execute 6 sec latency 84.210 ms 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.937 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi116 to 1936M 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.938 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Adjusting osd_memory_target on smithi118 to 2704M 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.939 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.940 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.940 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.940 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.940 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.940 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:16 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:16.971 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:37:16.971 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (7m) 2s ago 8m 26.7M - 0.23.0 ba2b418f427c 2e1af9a638b9 2024-09-20T19:37:16.971 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (8m) 2s ago 8m 6413k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:37:16.971 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (8m) 5s ago 8m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:37:16.971 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (7m) 2s ago 8m 53.6M - dad864ee21e9 564eedfb2837 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (2m) 2s ago 2m 34.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (2m) 5s ago 2m 12.9M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283 running (9m) 2s ago 9m 179M - 17.2.0 e1d6a67b021e 401b463eca11 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 running (36s) 5s ago 8m 474M - 19.3.0-5127-ge8baca1e 82f5f45939a4 4944e7a31ce8 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (9m) 2s ago 9m 47.0M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (7m) 5s ago 7m 36.3M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (2m) 2s ago 2m 229M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (2m) 5s ago 2m 77.0M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (8m) 2s ago 8m 22.6M - 1.3.1 1dbe0e931976 b18146bfd761 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 5s ago 7m 18.8M - 1.3.1 1dbe0e931976 2e0ae7aea427 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (7m) 2s ago 7m 274M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (6m) 2s ago 6m 202M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (6m) 2s ago 6m 165M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (5m) 2s ago 5m 218M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:37:16.972 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (5m) 5s ago 5m 241M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:37:16.973 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (4m) 5s ago 4m 258M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:37:16.973 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (4m) 5s ago 4m 245M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:37:16.973 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (4m) 5s ago 4m 333M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:37:16.973 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (7m) 2s ago 7m 64.8M - 2.33.4 514e6a882f6e b631805c5975 2024-09-20T19:37:17.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi116 to 1936M 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Adjusting osd_memory_target on smithi118 to 2704M 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:37:17.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.218 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.219 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.219 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.219 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:16 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:17.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 64697 14.18 MB/sec execute 7 sec latency 69.145 ms 2024-09-20T19:37:17.535 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:37:17.535 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:37:17.535 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:37:17.535 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:17.535 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:37:17.535 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T19:37:17.535 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:37:17.536 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:37:17.947 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:37:17.947 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:17.947 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:37:17.947 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: Reconfiguring alertmanager.smithi116 (dependencies changed)... 2024-09-20T19:37:17.947 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: Deploying daemon alertmanager.smithi116 on smithi116 2024-09-20T19:37:17.947 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: from='client.24435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:17.947 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.5 MiB/s wr, 679 op/s 2024-09-20T19:37:17.948 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:17 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1244306579' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:37:18.013 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:37:18.013 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:37:18.013 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:37:18.013 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:37:18.013 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [], 2024-09-20T19:37:18.013 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T19:37:18.013 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "", 2024-09-20T19:37:18.014 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:37:18.014 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:37:18.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:37:18.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:18.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:37:18.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: Reconfiguring alertmanager.smithi116 (dependencies changed)... 2024-09-20T19:37:18.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: Deploying daemon alertmanager.smithi116 on smithi116 2024-09-20T19:37:18.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: from='client.24435 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:18.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.5 MiB/s wr, 679 op/s 2024-09-20T19:37:18.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:17 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1244306579' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:37:18.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 65257 13.28 MB/sec execute 8 sec latency 36.090 ms 2024-09-20T19:37:18.562 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:37:18.562 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:37:18.562 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:37:19.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:19 smithi116 ceph-mon[23713]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:19.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:19 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2049130993' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:37:19.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 65807 13.11 MB/sec execute 9 sec latency 30.062 ms 2024-09-20T19:37:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:19 smithi118 ceph-mon[25856]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:19 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2049130993' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:37:20.234 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:20 smithi116 ceph-mon[23713]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 621 op/s 2024-09-20T19:37:20.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 66622 14.63 MB/sec execute 10 sec latency 25.160 ms 2024-09-20T19:37:20.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:20 smithi118 ceph-mon[25856]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 621 op/s 2024-09-20T19:37:21.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 67297 15.47 MB/sec execute 11 sec latency 23.675 ms 2024-09-20T19:37:21.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:21 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:21.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:21 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:21.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:21 smithi116 ceph-mon[23713]: pgmap v11: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 621 op/s 2024-09-20T19:37:22.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:21 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:22.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:21 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:22.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:21 smithi118 ceph-mon[25856]: pgmap v11: 97 pgs: 97 active+clean; 338 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 621 op/s 2024-09-20T19:37:22.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 67847 15.48 MB/sec execute 12 sec latency 64.913 ms 2024-09-20T19:37:23.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 68413 15.30 MB/sec execute 13 sec latency 23.311 ms 2024-09-20T19:37:24.263 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:24 smithi116 ceph-mon[23713]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 9.5 MiB/s wr, 973 op/s 2024-09-20T19:37:24.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 69107 14.92 MB/sec execute 14 sec latency 31.440 ms 2024-09-20T19:37:24.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:24 smithi118 ceph-mon[25856]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 9.5 MiB/s wr, 973 op/s 2024-09-20T19:37:25.266 INFO:teuthology.orchestra.run.smithi116.stdout: 5 69814 15.41 MB/sec execute 15 sec latency 19.788 ms 2024-09-20T19:37:26.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 70487 15.72 MB/sec execute 16 sec latency 19.264 ms 2024-09-20T19:37:26.299 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:26 smithi116 ceph-mon[23713]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 741 op/s 2024-09-20T19:37:26.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:26 smithi118 ceph-mon[25856]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 741 op/s 2024-09-20T19:37:27.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 71022 15.88 MB/sec execute 17 sec latency 33.137 ms 2024-09-20T19:37:28.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 71527 15.74 MB/sec execute 18 sec latency 26.878 ms 2024-09-20T19:37:28.318 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:28 smithi116 ceph-mon[23713]: pgmap v14: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.11k op/s 2024-09-20T19:37:28.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:28 smithi118 ceph-mon[25856]: pgmap v14: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.11k op/s 2024-09-20T19:37:29.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 72082 15.53 MB/sec execute 19 sec latency 58.129 ms 2024-09-20T19:37:30.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 72725 15.34 MB/sec execute 20 sec latency 62.068 ms 2024-09-20T19:37:30.337 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:30 smithi116 ceph-mon[23713]: pgmap v15: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-20T19:37:30.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:30 smithi118 ceph-mon[25856]: pgmap v15: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-20T19:37:31.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 73170 15.22 MB/sec execute 21 sec latency 89.591 ms 2024-09-20T19:37:32.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 73626 14.97 MB/sec execute 22 sec latency 51.533 ms 2024-09-20T19:37:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:32 smithi116 ceph-mon[23713]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-20T19:37:32.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:32 smithi118 ceph-mon[25856]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-20T19:37:33.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 74003 15.01 MB/sec execute 23 sec latency 46.009 ms 2024-09-20T19:37:34.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 74323 14.91 MB/sec execute 24 sec latency 52.957 ms 2024-09-20T19:37:34.477 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:34 smithi116 ceph-mon[23713]: pgmap v17: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-09-20T19:37:34.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:34 smithi118 ceph-mon[25856]: pgmap v17: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.04k op/s 2024-09-20T19:37:35.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 74643 14.69 MB/sec execute 25 sec latency 50.520 ms 2024-09-20T19:37:36.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 74940 14.49 MB/sec execute 26 sec latency 115.418 ms 2024-09-20T19:37:36.309 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:36 smithi116 ceph-mon[23713]: pgmap v18: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 689 op/s 2024-09-20T19:37:36.309 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:36 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:36.309 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:36 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:36.309 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:36 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:36.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:36 smithi118 ceph-mon[25856]: pgmap v18: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 689 op/s 2024-09-20T19:37:36.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:36 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:36.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:36 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:36.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:36 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:37.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 75113 14.12 MB/sec execute 27 sec latency 66.667 ms 2024-09-20T19:37:37.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:37 smithi116 ceph-mon[23713]: Reconfiguring grafana.smithi116 (dependencies changed)... 2024-09-20T19:37:37.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:37 smithi116 ceph-mon[23713]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-20T19:37:37.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:37 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:37.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:37 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:37.481 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:37 smithi118 ceph-mon[25856]: Reconfiguring grafana.smithi116 (dependencies changed)... 2024-09-20T19:37:37.481 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:37 smithi118 ceph-mon[25856]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-20T19:37:37.481 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:37 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:37.481 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:37 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:38.267 INFO:teuthology.orchestra.run.smithi116.stdout: 5 75353 13.80 MB/sec execute 28 sec latency 87.978 ms 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:38 smithi116 ceph-mon[23713]: dashboard set-grafana-api-ssl-verify failed: command not known retval: -22 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Traceback (most recent call last): 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1168, in _check_daemons 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2602, in _daemon_action 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: daemon_spec = self.cephadm_services[daemon_type_to_service( 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/usr/share/ceph/mgr/cephadm/services/monitoring.py", line 25, in prepare_create 2024-09-20T19:37:38.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec) 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/usr/share/ceph/mgr/cephadm/services/monitoring.py", line 129, in generate_config 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: cert, pkey = self.prepare_certificates(daemon_spec) 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/usr/share/ceph/mgr/cephadm/services/monitoring.py", line 196, in prepare_certificates 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self.mgr.check_mon_command({ 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/usr/share/ceph/mgr/mgr_module.py", line 1747, in check_mon_command 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: raise MonCommandFailed(f'{cmd_dict["prefix"]} failed: {r.stderr} retval: {r.retval}') 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: mgr_module.MonCommandFailed: dashboard set-grafana-api-ssl-verify failed: command not known retval: -22 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:38 smithi116 ceph-mon[23713]: Reconfiguring node-exporter.smithi116 (dependencies changed)... 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:38 smithi116 ceph-mon[23713]: Deploying daemon node-exporter.smithi116 on smithi116 2024-09-20T19:37:38.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:38 smithi116 ceph-mon[23713]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 890 op/s 2024-09-20T19:37:38.510 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:38 smithi118 ceph-mon[25856]: dashboard set-grafana-api-ssl-verify failed: command not known retval: -22 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Traceback (most recent call last): 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1168, in _check_daemons 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2602, in _daemon_action 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: daemon_spec = self.cephadm_services[daemon_type_to_service( 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/usr/share/ceph/mgr/cephadm/services/monitoring.py", line 25, in prepare_create 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec) 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/usr/share/ceph/mgr/cephadm/services/monitoring.py", line 129, in generate_config 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: cert, pkey = self.prepare_certificates(daemon_spec) 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/usr/share/ceph/mgr/cephadm/services/monitoring.py", line 196, in prepare_certificates 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.mgr.check_mon_command({ 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/usr/share/ceph/mgr/mgr_module.py", line 1747, in check_mon_command 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: raise MonCommandFailed(f'{cmd_dict["prefix"]} failed: {r.stderr} retval: {r.retval}') 2024-09-20T19:37:38.511 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: mgr_module.MonCommandFailed: dashboard set-grafana-api-ssl-verify failed: command not known retval: -22 2024-09-20T19:37:38.512 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:38 smithi118 ceph-mon[25856]: Reconfiguring node-exporter.smithi116 (dependencies changed)... 2024-09-20T19:37:38.512 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:38 smithi118 ceph-mon[25856]: Deploying daemon node-exporter.smithi116 on smithi116 2024-09-20T19:37:38.512 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:38 smithi118 ceph-mon[25856]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 890 op/s 2024-09-20T19:37:39.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 75681 13.48 MB/sec execute 29 sec latency 72.934 ms 2024-09-20T19:37:40.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 76050 13.19 MB/sec execute 30 sec latency 64.287 ms 2024-09-20T19:37:40.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:40 smithi116 ceph-mon[23713]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 522 op/s 2024-09-20T19:37:40.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:40 smithi118 ceph-mon[25856]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 522 op/s 2024-09-20T19:37:41.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 76375 13.03 MB/sec execute 31 sec latency 67.967 ms 2024-09-20T19:37:42.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 77056 13.25 MB/sec execute 32 sec latency 54.961 ms 2024-09-20T19:37:42.456 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:42 smithi116 ceph-mon[23713]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 522 op/s 2024-09-20T19:37:42.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:42 smithi118 ceph-mon[25856]: pgmap v21: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 522 op/s 2024-09-20T19:37:43.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 77419 13.13 MB/sec execute 33 sec latency 69.263 ms 2024-09-20T19:37:44.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 77895 13.35 MB/sec execute 34 sec latency 54.042 ms 2024-09-20T19:37:44.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:44 smithi116 ceph-mon[23713]: pgmap v22: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-09-20T19:37:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:44 smithi118 ceph-mon[25856]: pgmap v22: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-09-20T19:37:45.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 78339 13.28 MB/sec execute 35 sec latency 73.835 ms 2024-09-20T19:37:46.172 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:46 smithi116 ceph-mon[23713]: pgmap v23: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 394 op/s 2024-09-20T19:37:46.172 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:46 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:46.172 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:46 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:46.172 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:46 smithi116 ceph-mon[23713]: Reconfiguring prometheus.smithi116 (dependencies changed)... 2024-09-20T19:37:46.172 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:46 smithi116 ceph-mon[23713]: Deploying daemon prometheus.smithi116 on smithi116 2024-09-20T19:37:46.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 78595 13.21 MB/sec execute 36 sec latency 65.596 ms 2024-09-20T19:37:46.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:46 smithi118 ceph-mon[25856]: pgmap v23: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.2 MiB/s wr, 394 op/s 2024-09-20T19:37:46.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:46 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:46.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:46 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:37:46.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:46 smithi118 ceph-mon[25856]: Reconfiguring prometheus.smithi116 (dependencies changed)... 2024-09-20T19:37:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:46 smithi118 ceph-mon[25856]: Deploying daemon prometheus.smithi116 on smithi116 2024-09-20T19:37:47.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 78845 13.02 MB/sec execute 37 sec latency 64.405 ms 2024-09-20T19:37:48.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:48 smithi118 ceph-mon[25856]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 651 op/s 2024-09-20T19:37:48.228 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:48 smithi116 ceph-mon[23713]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 651 op/s 2024-09-20T19:37:48.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 79393 12.84 MB/sec execute 38 sec latency 40.600 ms 2024-09-20T19:37:49.112 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:37:49.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 79789 12.72 MB/sec execute 39 sec latency 67.299 ms 2024-09-20T19:37:50.206 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:37:50.206 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 starting - - - - 2024-09-20T19:37:50.206 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (9m) 35s ago 9m 6413k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (8m) 38s ago 8m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (7m) 35s ago 8m 53.6M - dad864ee21e9 564eedfb2837 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (3m) 35s ago 3m 34.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (3m) 38s ago 3m 12.9M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283 running (10m) 35s ago 10m 179M - 17.2.0 e1d6a67b021e 401b463eca11 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 running (69s) 38s ago 8m 474M - 19.3.0-5127-ge8baca1e 82f5f45939a4 4944e7a31ce8 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (10m) 35s ago 10m 47.0M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (8m) 38s ago 8m 36.3M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (2m) 35s ago 2m 229M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (2m) 38s ago 2m 77.0M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 starting - - - - 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (8m) 38s ago 8m 18.8M - 1.3.1 1dbe0e931976 2e0ae7aea427 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (7m) 35s ago 7m 274M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:37:50.207 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (7m) 35s ago 7m 202M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:37:50.208 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (6m) 35s ago 6m 165M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:37:50.208 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (6m) 35s ago 6m 218M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:37:50.208 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (5m) 38s ago 5m 241M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:37:50.208 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (5m) 38s ago 5m 258M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:37:50.208 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (5m) 38s ago 5m 245M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:37:50.208 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (4m) 38s ago 4m 333M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:37:50.208 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (8m) 35s ago 8m 64.8M - 2.33.4 514e6a882f6e b631805c5975 2024-09-20T19:37:50.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:50 smithi116 ceph-mon[23713]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-20T19:37:50.258 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:50 smithi116 ceph-mon[23713]: from='client.24451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:50.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 80181 12.78 MB/sec execute 40 sec latency 48.910 ms 2024-09-20T19:37:50.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:50 smithi118 ceph-mon[25856]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-20T19:37:50.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:50 smithi118 ceph-mon[25856]: from='client.24451 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:37:50.883 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:37:50.884 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:37:51.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 80620 12.67 MB/sec execute 41 sec latency 61.952 ms 2024-09-20T19:37:51.294 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:51 smithi116 ceph-mon[23713]: from='client.24455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:51.294 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:51 smithi116 ceph-mon[23713]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:51.294 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:51 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:51.294 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:51 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3879120920' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [], 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "", 2024-09-20T19:37:51.468 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:37:51.469 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:37:51.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:51 smithi118 ceph-mon[25856]: from='client.24455 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:51.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:51 smithi118 ceph-mon[25856]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:51.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:51 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:37:51.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:51 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3879120920' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:37:52.002 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:37:52.002 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:37:52.002 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:37:52.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 80949 12.58 MB/sec execute 42 sec latency 87.873 ms 2024-09-20T19:37:52.317 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:52 smithi116 ceph-mon[23713]: pgmap v26: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-20T19:37:52.317 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:52 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/1191263726' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:37:52.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:52 smithi118 ceph-mon[25856]: pgmap v26: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-20T19:37:52.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:52 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/1191263726' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:37:53.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 81667 12.89 MB/sec execute 43 sec latency 29.999 ms 2024-09-20T19:37:53.364 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:53 smithi116 ceph-mon[23713]: from='client.24467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:53.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:53 smithi118 ceph-mon[25856]: from='client.24467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:37:54.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 82091 12.94 MB/sec execute 44 sec latency 70.895 ms 2024-09-20T19:37:54.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:54 smithi116 ceph-mon[23713]: pgmap v27: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 688 op/s 2024-09-20T19:37:54.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:54 smithi118 ceph-mon[25856]: pgmap v27: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 688 op/s 2024-09-20T19:37:55.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 82589 12.85 MB/sec execute 45 sec latency 31.132 ms 2024-09-20T19:37:56.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 83378 12.92 MB/sec execute 46 sec latency 28.901 ms 2024-09-20T19:37:56.413 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:56 smithi116 ceph-mon[23713]: pgmap v28: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 494 op/s 2024-09-20T19:37:56.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:56 smithi118 ceph-mon[25856]: pgmap v28: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 494 op/s 2024-09-20T19:37:57.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 83966 13.00 MB/sec execute 47 sec latency 39.058 ms 2024-09-20T19:37:58.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 84400 12.95 MB/sec execute 48 sec latency 83.037 ms 2024-09-20T19:37:58.466 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:37:58 smithi116 ceph-mon[23713]: pgmap v29: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 812 op/s 2024-09-20T19:37:58.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:37:58 smithi118 ceph-mon[25856]: pgmap v29: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 812 op/s 2024-09-20T19:37:59.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 84697 12.92 MB/sec execute 49 sec latency 61.369 ms 2024-09-20T19:38:00.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 84987 12.91 MB/sec execute 50 sec latency 80.425 ms 2024-09-20T19:38:00.302 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:00 smithi116 ceph-mon[23713]: pgmap v30: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-09-20T19:38:00.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:00 smithi118 ceph-mon[25856]: pgmap v30: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-09-20T19:38:01.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 85352 12.85 MB/sec execute 51 sec latency 47.618 ms 2024-09-20T19:38:02.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 85805 12.85 MB/sec execute 52 sec latency 73.437 ms 2024-09-20T19:38:02.315 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:02 smithi116 ceph-mon[23713]: pgmap v31: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-09-20T19:38:02.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:02 smithi118 ceph-mon[25856]: pgmap v31: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-09-20T19:38:03.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 86260 12.73 MB/sec execute 53 sec latency 33.659 ms 2024-09-20T19:38:04.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 86674 12.65 MB/sec execute 54 sec latency 69.485 ms 2024-09-20T19:38:04.347 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:04 smithi116 ceph-mon[23713]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 794 op/s 2024-09-20T19:38:04.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:04 smithi118 ceph-mon[25856]: pgmap v32: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 794 op/s 2024-09-20T19:38:05.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 87130 12.67 MB/sec execute 55 sec latency 44.019 ms 2024-09-20T19:38:06.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 87563 12.61 MB/sec execute 56 sec latency 50.923 ms 2024-09-20T19:38:06.374 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:06 smithi116 ceph-mon[23713]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 556 op/s 2024-09-20T19:38:06.374 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:06 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:06.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:06 smithi118 ceph-mon[25856]: pgmap v33: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 556 op/s 2024-09-20T19:38:06.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:06 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:07.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 88015 12.60 MB/sec execute 57 sec latency 42.166 ms 2024-09-20T19:38:07.645 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:07 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:07.645 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:07 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:07.645 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:07 smithi116 ceph-mon[23713]: Reconfiguring node-exporter.smithi118 (dependencies changed)... 2024-09-20T19:38:07.645 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:07 smithi116 ceph-mon[23713]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-20T19:38:07.645 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:07 smithi116 ceph-mon[23713]: pgmap v34: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 828 op/s 2024-09-20T19:38:07.833 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:07 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:07.834 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:07 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:07.834 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:07 smithi118 ceph-mon[25856]: Reconfiguring node-exporter.smithi118 (dependencies changed)... 2024-09-20T19:38:07.834 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:07 smithi118 ceph-mon[25856]: Deploying daemon node-exporter.smithi118 on smithi118 2024-09-20T19:38:07.834 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:07 smithi118 ceph-mon[25856]: pgmap v34: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 828 op/s 2024-09-20T19:38:08.268 INFO:teuthology.orchestra.run.smithi116.stdout: 5 88484 12.72 MB/sec execute 58 sec latency 40.042 ms 2024-09-20T19:38:09.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 88900 12.78 MB/sec execute 59 sec latency 64.293 ms 2024-09-20T19:38:10.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 89345 12.73 MB/sec execute 60 sec latency 54.879 ms 2024-09-20T19:38:10.392 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:10 smithi116 ceph-mon[23713]: pgmap v35: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-09-20T19:38:10.528 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:10 smithi118 ceph-mon[25856]: pgmap v35: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-09-20T19:38:11.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 89645 12.61 MB/sec execute 61 sec latency 78.048 ms 2024-09-20T19:38:12.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 90197 12.60 MB/sec execute 62 sec latency 78.506 ms 2024-09-20T19:38:12.428 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:12 smithi116 ceph-mon[23713]: pgmap v36: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-09-20T19:38:12.428 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:12 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:12.428 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:12 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:12.428 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:12 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:12.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:12 smithi118 ceph-mon[25856]: pgmap v36: 97 pgs: 97 active+clean; 315 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-09-20T19:38:12.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:12 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:12.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:12 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:12.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:12 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:13.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 90548 12.50 MB/sec execute 63 sec latency 59.947 ms 2024-09-20T19:38:13.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:13 smithi116 ceph-mon[23713]: Failed to get Dashboard config for AlertManager: dashboard get-alertmanager-api-host failed: command not known retval: -22 2024-09-20T19:38:13.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:13 smithi116 ceph-mon[23713]: Failed to get Dashboard config for Prometheus: dashboard get-prometheus-api-host failed: command not known retval: -22 2024-09-20T19:38:13.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:13 smithi116 ceph-mon[23713]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:13.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:13 smithi116 ceph-mon[23713]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:13.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:13 smithi118 ceph-mon[25856]: Failed to get Dashboard config for AlertManager: dashboard get-alertmanager-api-host failed: command not known retval: -22 2024-09-20T19:38:13.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:13 smithi118 ceph-mon[25856]: Failed to get Dashboard config for Prometheus: dashboard get-prometheus-api-host failed: command not known retval: -22 2024-09-20T19:38:13.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:13 smithi118 ceph-mon[25856]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:13.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:13 smithi118 ceph-mon[25856]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:14.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 90834 12.40 MB/sec execute 64 sec latency 91.849 ms 2024-09-20T19:38:14.491 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:14 smithi116 ceph-mon[23713]: pgmap v37: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 763 op/s 2024-09-20T19:38:14.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:14 smithi118 ceph-mon[25856]: pgmap v37: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 763 op/s 2024-09-20T19:38:15.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 91167 12.34 MB/sec execute 65 sec latency 87.393 ms 2024-09-20T19:38:15.773 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: Upgrade: Updating mgr.smithi116.poxcrq 2024-09-20T19:38:15.774 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:15.774 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:38:15.774 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:38:15.774 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:38:15.774 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:38:15.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: Deploying daemon mgr.smithi116.poxcrq on smithi116 2024-09-20T19:38:15.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:15 smithi116 ceph-mon[23713]: pgmap v38: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-09-20T19:38:15.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: Upgrade: Updating mgr.smithi116.poxcrq 2024-09-20T19:38:15.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:15.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:38:15.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:38:15.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:38:15.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:38:15.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: Deploying daemon mgr.smithi116.poxcrq on smithi116 2024-09-20T19:38:15.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:15 smithi118 ceph-mon[25856]: pgmap v38: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-09-20T19:38:16.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 91546 12.36 MB/sec execute 66 sec latency 77.146 ms 2024-09-20T19:38:17.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 91874 12.35 MB/sec execute 67 sec latency 59.981 ms 2024-09-20T19:38:18.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 92253 12.40 MB/sec execute 68 sec latency 57.693 ms 2024-09-20T19:38:18.323 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:18 smithi116 ceph-mon[23713]: pgmap v39: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 748 op/s 2024-09-20T19:38:18.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:18 smithi118 ceph-mon[25856]: pgmap v39: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 748 op/s 2024-09-20T19:38:19.269 INFO:teuthology.orchestra.run.smithi116.stdout: 5 92472 12.31 MB/sec execute 69 sec latency 117.890 ms 2024-09-20T19:38:20.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 92698 12.24 MB/sec execute 70 sec latency 72.556 ms 2024-09-20T19:38:20.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:20 smithi116 ceph-mon[23713]: pgmap v40: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-09-20T19:38:20.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:20 smithi118 ceph-mon[25856]: pgmap v40: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-09-20T19:38:21.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 92908 12.11 MB/sec execute 71 sec latency 81.712 ms 2024-09-20T19:38:21.653 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:21 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.653 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:21 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.654 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:21 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:21.654 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:21 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.654 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:21 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.654 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:21 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:21.654 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:21 smithi116 ceph-mon[23713]: pgmap v41: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-09-20T19:38:21.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:21 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:21 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:21 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:21 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:21 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:21 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:21 smithi118 ceph-mon[25856]: pgmap v41: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 476 op/s 2024-09-20T19:38:22.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 93457 12.05 MB/sec execute 72 sec latency 57.343 ms 2024-09-20T19:38:22.602 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:38:23.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 93936 12.07 MB/sec execute 73 sec latency 50.399 ms 2024-09-20T19:38:23.644 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:38:23.644 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 starting - - - - 2024-09-20T19:38:23.644 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (9m) 68s ago 9m 6413k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:38:23.644 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (9m) 72s ago 9m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:38:23.644 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (8m) 68s ago 9m 53.6M - dad864ee21e9 564eedfb2837 2024-09-20T19:38:23.644 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (3m) 68s ago 3m 34.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:38:23.644 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (3m) 72s ago 3m 12.9M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 starting - - - - 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 running (102s) 72s ago 9m 474M - 19.3.0-5127-ge8baca1e 82f5f45939a4 4944e7a31ce8 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (11m) 68s ago 11m 47.0M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (9m) 72s ago 9m 36.3M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (3m) 68s ago 3m 229M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (3m) 72s ago 3m 77.0M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 starting - - - - 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 starting - - - - 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (8m) 68s ago 8m 274M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (7m) 68s ago 7m 202M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (7m) 68s ago 7m 165M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (6m) 68s ago 6m 218M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (6m) 72s ago 6m 241M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:38:23.645 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (6m) 72s ago 6m 258M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:38:23.646 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (5m) 72s ago 5m 245M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:38:23.646 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (5m) 72s ago 5m 333M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:38:23.646 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 starting - - - - 2024-09-20T19:38:24.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 94365 12.02 MB/sec execute 74 sec latency 52.671 ms 2024-09-20T19:38:24.319 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:38:24.320 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:38:24.321 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:24.321 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:38:24.321 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T19:38:24.321 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:38:24.321 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:38:24.321 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:38:24.446 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:24 smithi116 ceph-mon[23713]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:24.446 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:24 smithi116 ceph-mon[23713]: pgmap v42: 97 pgs: 97 active+clean; 321 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.1 MiB/s wr, 650 op/s 2024-09-20T19:38:24.446 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:24 smithi116 ceph-mon[23713]: from='client.24483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:24.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:24 smithi118 ceph-mon[25856]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:24.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:24 smithi118 ceph-mon[25856]: pgmap v42: 97 pgs: 97 active+clean; 321 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.1 MiB/s wr, 650 op/s 2024-09-20T19:38:24.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:24 smithi118 ceph-mon[25856]: from='client.24483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:24.911 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:38:24.911 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:38:24.911 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:38:24.911 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:38:24.911 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [], 2024-09-20T19:38:24.911 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "", 2024-09-20T19:38:24.911 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading mgr daemons", 2024-09-20T19:38:24.912 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:38:24.912 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:38:25.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 94707 11.98 MB/sec execute 75 sec latency 40.932 ms 2024-09-20T19:38:25.456 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:25 smithi116 ceph-mon[23713]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:25.456 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:25 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:25.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:25 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:25.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:25 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/2813347906' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:38:25.658 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:38:25.659 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:38:25.659 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:38:25.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:25 smithi118 ceph-mon[25856]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:25.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:25 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:25.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:25 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:25.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:25 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2813347906' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:38:26.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 95036 11.96 MB/sec execute 76 sec latency 42.263 ms 2024-09-20T19:38:26.491 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:26 smithi116 ceph-mon[23713]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:26.491 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:26 smithi116 ceph-mon[23713]: pgmap v43: 97 pgs: 97 active+clean; 321 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 397 op/s 2024-09-20T19:38:26.491 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:26 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3862332645' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:38:26.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:26 smithi118 ceph-mon[25856]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:26.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:26 smithi118 ceph-mon[25856]: pgmap v43: 97 pgs: 97 active+clean; 321 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 397 op/s 2024-09-20T19:38:26.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:26 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3862332645' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:38:27.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 95359 11.98 MB/sec execute 77 sec latency 53.058 ms 2024-09-20T19:38:28.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 95754 12.02 MB/sec execute 78 sec latency 55.882 ms 2024-09-20T19:38:28.523 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:28 smithi116 ceph-mon[23713]: pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 647 op/s 2024-09-20T19:38:28.523 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:28 smithi116 ceph-mon[23713]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:38:28.523 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:28 smithi116 ceph-mon[23713]: Standby manager daemon smithi116.poxcrq restarted 2024-09-20T19:38:28.523 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:28 smithi116 ceph-mon[23713]: Standby manager daemon smithi116.poxcrq started 2024-09-20T19:38:28.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:28 smithi118 ceph-mon[25856]: pgmap v44: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 647 op/s 2024-09-20T19:38:28.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:28 smithi118 ceph-mon[25856]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:38:28.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:28 smithi118 ceph-mon[25856]: Standby manager daemon smithi116.poxcrq restarted 2024-09-20T19:38:28.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:28 smithi118 ceph-mon[25856]: Standby manager daemon smithi116.poxcrq started 2024-09-20T19:38:29.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 96126 12.00 MB/sec execute 79 sec latency 66.732 ms 2024-09-20T19:38:29.537 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:29 smithi116 ceph-mon[23713]: mgrmap e28: smithi118.rmiazv(active, since 84s), standbys: smithi116.poxcrq 2024-09-20T19:38:29.537 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:29 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:29.537 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:29 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:29 smithi118 ceph-mon[25856]: mgrmap e28: smithi118.rmiazv(active, since 84s), standbys: smithi116.poxcrq 2024-09-20T19:38:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:29 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:29 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:30.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 96387 11.93 MB/sec execute 80 sec latency 51.425 ms 2024-09-20T19:38:30.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:30 smithi116 ceph-mon[23713]: pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-20T19:38:30.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:30 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:30.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:30 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:30.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:30 smithi118 ceph-mon[25856]: pgmap v45: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-20T19:38:30.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:30 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:30.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:30 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:31.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 96697 11.85 MB/sec execute 81 sec latency 68.771 ms 2024-09-20T19:38:31.596 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:31 smithi116 ceph-mon[23713]: pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-20T19:38:31.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:31 smithi118 ceph-mon[25856]: pgmap v46: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-20T19:38:32.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 96948 11.73 MB/sec execute 82 sec latency 100.646 ms 2024-09-20T19:38:33.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 97321 11.72 MB/sec execute 83 sec latency 61.738 ms 2024-09-20T19:38:34.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 97731 11.66 MB/sec execute 84 sec latency 42.831 ms 2024-09-20T19:38:34.420 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:34 smithi116 ceph-mon[23713]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 622 op/s 2024-09-20T19:38:34.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:34 smithi118 ceph-mon[25856]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 622 op/s 2024-09-20T19:38:35.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 98145 11.65 MB/sec execute 85 sec latency 69.252 ms 2024-09-20T19:38:36.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 98539 11.66 MB/sec execute 86 sec latency 44.151 ms 2024-09-20T19:38:36.455 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:36 smithi116 ceph-mon[23713]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-20T19:38:36.455 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:36 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:36.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:36 smithi118 ceph-mon[25856]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-20T19:38:36.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:36 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:37.270 INFO:teuthology.orchestra.run.smithi116.stdout: 5 98901 11.70 MB/sec execute 87 sec latency 44.251 ms 2024-09-20T19:38:37.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:37 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:37.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:37 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:37.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:37 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:38:37.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:37 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:38:37.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:37 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:37.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:37 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:37.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:37 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:37.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:37 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:37.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:37 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:38:37.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:37 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:38:37.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:37 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:37.616 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:37 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:38.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 99563 11.83 MB/sec execute 88 sec latency 19.179 ms 2024-09-20T19:38:38.467 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:38 smithi116 ceph-mon[23713]: Reconfiguring grafana.smithi116 (dependencies changed)... 2024-09-20T19:38:38.467 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:38 smithi116 ceph-mon[23713]: Reconfiguring daemon grafana.smithi116 on smithi116 2024-09-20T19:38:38.468 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:38 smithi116 ceph-mon[23713]: pgmap v49: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 669 op/s 2024-09-20T19:38:38.468 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:38 smithi116 ceph-mon[23713]: mgrmap e29: smithi118.rmiazv(active, since 93s), standbys: smithi116.poxcrq 2024-09-20T19:38:38.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:38 smithi118 ceph-mon[25856]: Reconfiguring grafana.smithi116 (dependencies changed)... 2024-09-20T19:38:38.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:38 smithi118 ceph-mon[25856]: Reconfiguring daemon grafana.smithi116 on smithi116 2024-09-20T19:38:38.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:38 smithi118 ceph-mon[25856]: pgmap v49: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 669 op/s 2024-09-20T19:38:38.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:38 smithi118 ceph-mon[25856]: mgrmap e29: smithi118.rmiazv(active, since 93s), standbys: smithi116.poxcrq 2024-09-20T19:38:39.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 99987 11.80 MB/sec execute 89 sec latency 31.641 ms 2024-09-20T19:38:40.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 100386 11.75 MB/sec execute 90 sec latency 40.721 ms 2024-09-20T19:38:40.465 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:40 smithi116 ceph-mon[23713]: pgmap v50: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-20T19:38:40.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:40 smithi118 ceph-mon[25856]: pgmap v50: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-20T19:38:41.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 100903 11.75 MB/sec execute 91 sec latency 32.676 ms 2024-09-20T19:38:42.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 101516 11.78 MB/sec execute 92 sec latency 55.383 ms 2024-09-20T19:38:42.489 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:42 smithi116 ceph-mon[23713]: pgmap v51: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 430 op/s 2024-09-20T19:38:42.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:42 smithi118 ceph-mon[25856]: pgmap v51: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 430 op/s 2024-09-20T19:38:43.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 102002 11.74 MB/sec execute 93 sec latency 28.176 ms 2024-09-20T19:38:44.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 102480 11.86 MB/sec execute 94 sec latency 32.719 ms 2024-09-20T19:38:44.408 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:44 smithi116 ceph-mon[23713]: pgmap v52: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.3 MiB/s wr, 699 op/s 2024-09-20T19:38:44.409 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:44 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:44 smithi118 ceph-mon[25856]: pgmap v52: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.3 MiB/s wr, 699 op/s 2024-09-20T19:38:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:44 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:45.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 102977 11.94 MB/sec execute 95 sec latency 38.824 ms 2024-09-20T19:38:45.535 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: from='client.24501 -' entity='client.crash.smithi116' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:45.535 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: pgmap v53: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-09-20T19:38:45.535 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH) 2024-09-20T19:38:45.535 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:45.535 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:45.536 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:45.536 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr fail", "who": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:38:45.536 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr fail", "who": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:38:45.536 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:45 smithi116 ceph-mon[23713]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T19:38:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: from='client.24501 -' entity='client.crash.smithi116' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: pgmap v53: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-09-20T19:38:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH) 2024-09-20T19:38:45.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' 2024-09-20T19:38:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: from='mgr.14662 172.21.15.118:0/1233996048' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr fail", "who": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:38:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd=[{"prefix": "mgr fail", "who": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:38:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:45 smithi118 ceph-mon[25856]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T19:38:46.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 103302 11.90 MB/sec execute 96 sec latency 44.151 ms 2024-09-20T19:38:46.558 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: Failed to get Dashboard config for Grafana: dashboard get-grafana-api-url failed: command not known retval: -22 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: Failing over to other MGR 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd='[{"prefix": "mgr fail", "who": "smithi118.rmiazv"}]': finished 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: mgrmap e30: smithi116.poxcrq(active, starting, since 0.707299s) 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"}]: dispatch 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"}]: dispatch 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:38:46.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:38:46.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:46 smithi116 ceph-mon[23713]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:38:46.684 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: Failed to get Dashboard config for Grafana: dashboard get-grafana-api-url failed: command not known retval: -22 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: Upgrade: Need to upgrade myself (mgr.smithi118.rmiazv) 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: Failing over to other MGR 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14662 ' entity='mgr.smithi118.rmiazv' cmd='[{"prefix": "mgr fail", "who": "smithi118.rmiazv"}]': finished 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: mgrmap e30: smithi116.poxcrq(active, starting, since 0.707299s) 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi116"}]: dispatch 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata", "id": "smithi118"}]: dispatch 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"}]: dispatch 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"}]: dispatch 2024-09-20T19:38:46.685 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T19:38:46.686 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:46 smithi118 ceph-mon[25856]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:38:47.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 103544 11.82 MB/sec execute 97 sec latency 44.725 ms 2024-09-20T19:38:47.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:47.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T19:38:47.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:47.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:47.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:38:47.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:38:47.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:38:47.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:38:47.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: mgrmap e31: smithi116.poxcrq(active, since 1.71077s) 2024-09-20T19:38:47.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T19:38:47.576 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:47 smithi116 ceph-mon[23713]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"}]: dispatch 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"}]: dispatch 2024-09-20T19:38:47.609 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: mgrmap e31: smithi116.poxcrq(active, since 1.71077s) 2024-09-20T19:38:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T19:38:47.610 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:47 smithi118 ceph-mon[25856]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T19:38:48.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 103851 11.75 MB/sec execute 98 sec latency 51.951 ms 2024-09-20T19:38:49.104 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:48 smithi116 ceph-mon[23713]: [20/Sep/2024:19:38:47] ENGINE Bus STARTING 2024-09-20T19:38:49.104 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:48 smithi116 ceph-mon[23713]: [20/Sep/2024:19:38:48] ENGINE Serving on https://172.21.15.116:7150 2024-09-20T19:38:49.104 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:48 smithi116 ceph-mon[23713]: [20/Sep/2024:19:38:48] ENGINE Error in HTTPServer.serve 2024-09-20T19:38:49.104 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Traceback (most recent call last): 2024-09-20T19:38:49.104 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T19:38:49.104 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._connections.run(self.expiration_interval) 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._run(expiration_interval) 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: s = self.context.wrap_socket( 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: return self.sslsocket_class._create( 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self.do_handshake() 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T19:38:49.105 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._sslobj.do_handshake() 2024-09-20T19:38:49.106 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T19:38:49.106 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:48 smithi116 ceph-mon[23713]: mgrmap e32: smithi116.poxcrq(active, since 3s) 2024-09-20T19:38:49.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:48 smithi118 ceph-mon[25856]: [20/Sep/2024:19:38:47] ENGINE Bus STARTING 2024-09-20T19:38:49.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:48 smithi118 ceph-mon[25856]: [20/Sep/2024:19:38:48] ENGINE Serving on https://172.21.15.116:7150 2024-09-20T19:38:49.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:48 smithi118 ceph-mon[25856]: [20/Sep/2024:19:38:48] ENGINE Error in HTTPServer.serve 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Traceback (most recent call last): 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._connections.run(self.expiration_interval) 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._run(expiration_interval) 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s = self.context.wrap_socket( 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: return self.sslsocket_class._create( 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T19:38:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.do_handshake() 2024-09-20T19:38:49.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T19:38:49.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._sslobj.do_handshake() 2024-09-20T19:38:49.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T19:38:49.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:48 smithi118 ceph-mon[25856]: mgrmap e32: smithi116.poxcrq(active, since 3s) 2024-09-20T19:38:49.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 104136 11.67 MB/sec execute 99 sec latency 58.656 ms 2024-09-20T19:38:50.113 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:49 smithi116 ceph-mon[23713]: [20/Sep/2024:19:38:48] ENGINE Serving on http://172.21.15.116:8765 2024-09-20T19:38:50.113 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:49 smithi116 ceph-mon[23713]: [20/Sep/2024:19:38:48] ENGINE Bus STARTED 2024-09-20T19:38:50.114 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:49 smithi116 ceph-mon[23713]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T19:38:50.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:49 smithi118 ceph-mon[25856]: [20/Sep/2024:19:38:48] ENGINE Serving on http://172.21.15.116:8765 2024-09-20T19:38:50.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:49 smithi118 ceph-mon[25856]: [20/Sep/2024:19:38:48] ENGINE Bus STARTED 2024-09-20T19:38:50.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:49 smithi118 ceph-mon[25856]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T19:38:50.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 104559 11.69 MB/sec execute 100 sec latency 44.275 ms 2024-09-20T19:38:51.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 105188 11.69 MB/sec execute 101 sec latency 38.119 ms 2024-09-20T19:38:51.377 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:51 smithi116 ceph-mon[23713]: mgrmap e33: smithi116.poxcrq(active, since 5s) 2024-09-20T19:38:51.377 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:51 smithi116 ceph-mon[23713]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T19:38:51.377 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:51 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:51.377 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:51 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:51.666 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:51 smithi118 ceph-mon[25856]: mgrmap e33: smithi116.poxcrq(active, since 5s) 2024-09-20T19:38:51.667 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:51 smithi118 ceph-mon[25856]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-20T19:38:51.667 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:51 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:51.667 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:51 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:52.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 105661 11.71 MB/sec execute 102 sec latency 67.022 ms 2024-09-20T19:38:52.646 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:52 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:52.646 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:52 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:52.646 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:52 smithi116 ceph-mon[23713]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:38:52.646 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:52 smithi116 ceph-mon[23713]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:38:52.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:52 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:52.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:52 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:52.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:52 smithi118 ceph-mon[25856]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:38:52.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:52 smithi118 ceph-mon[25856]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:38:53.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 106163 11.77 MB/sec execute 103 sec latency 83.523 ms 2024-09-20T19:38:53.655 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:53 smithi116 ceph-mon[23713]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 4.5 MiB/s wr, 555 op/s 2024-09-20T19:38:53.655 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:53 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rmiazv", "id": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:38:53.655 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:53 smithi116 ceph-mon[23713]: mgrmap e34: smithi116.poxcrq(active, since 7s), standbys: smithi118.rmiazv 2024-09-20T19:38:53.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:53 smithi118 ceph-mon[25856]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 4.5 MiB/s wr, 555 op/s 2024-09-20T19:38:53.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:53 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr metadata", "who": "smithi118.rmiazv", "id": "smithi118.rmiazv"}]: dispatch 2024-09-20T19:38:53.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:53 smithi118 ceph-mon[25856]: mgrmap e34: smithi116.poxcrq(active, since 7s), standbys: smithi118.rmiazv 2024-09-20T19:38:54.271 INFO:teuthology.orchestra.run.smithi116.stdout: 5 106584 11.83 MB/sec execute 104 sec latency 45.659 ms 2024-09-20T19:38:55.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 106922 11.81 MB/sec execute 105 sec latency 50.696 ms 2024-09-20T19:38:55.697 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:55 smithi116 ceph-mon[23713]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 3.3 MiB/s wr, 405 op/s 2024-09-20T19:38:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:55 smithi118 ceph-mon[25856]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 3.3 MiB/s wr, 405 op/s 2024-09-20T19:38:56.208 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:38:56.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 107298 11.76 MB/sec execute 106 sec latency 37.565 ms 2024-09-20T19:38:56.958 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:56 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:56.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:56 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:56.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:56 smithi116 ceph-mon[23713]: from='client.24531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:56.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:56 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:56.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:56 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:56.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:56 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:38:56.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:56 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:38:57.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:56 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:56 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:56 smithi118 ceph-mon[25856]: from='client.24531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:57.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:56 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:56 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:56 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:38:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:56 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:38:57.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 107570 11.67 MB/sec execute 107 sec latency 86.790 ms 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (82s) 1s ago 10m 15.4M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (10m) 1s ago 10m 6433k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (9m) 6s ago 9m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (13s) 1s ago 10m 38.4M - dad864ee21e9 8a163a77164a 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (4m) 1s ago 4m 36.3M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (4m) 6s ago 4m 13.5M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (37s) 1s ago 11m 468M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:8443,9283 running (2m) 6s ago 9m 105M - 19.3.0-5127-ge8baca1e 82f5f45939a4 4944e7a31ce8 2024-09-20T19:38:57.285 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (11m) 1s ago 11m 51.9M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (9m) 6s ago 9m 40.8M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (4m) 1s ago 3m 262M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (3m) 6s ago 3m 81.2M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (73s) 1s ago 10m 9.96M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (46s) 6s ago 9m 9.87M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (8m) 1s ago 8m 408M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (8m) 1s ago 8m 319M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (7m) 1s ago 7m 232M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (7m) 1s ago 7m 390M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (7m) 6s ago 7m 389M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (6m) 6s ago 6m 400M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (6m) 6s ago 6m 339M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (5m) 6s ago 5m 483M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:38:57.286 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (51s) 1s ago 9m 41.9M - 2.51.0 1d3b7f56885b d9987dd52dc1 2024-09-20T19:38:57.824 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:38:57.825 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:38:57.826 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-20T19:38:57.826 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:38:57.826 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:38:57.826 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:38:57.972 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.6 MiB/s wr, 319 op/s 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:57.973 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:57 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/3572595953' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.6 MiB/s wr, 319 op/s 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:58.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:58.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:38:58.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:57 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/3572595953' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:38:58.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 107910 11.64 MB/sec execute 108 sec latency 64.889 ms 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr" 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "2/23 daemons upgraded", 2024-09-20T19:38:58.282 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "", 2024-09-20T19:38:58.283 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:38:58.283 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:38:58.785 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:38:58.785 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:38:58.785 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:38:58.785 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:38:58.786 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:38:58.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:58 smithi116 ceph-mon[23713]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:58.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:58 smithi116 ceph-mon[23713]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:38:58.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:58 smithi116 ceph-mon[23713]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:38:58.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:58 smithi116 ceph-mon[23713]: Reconfiguring prometheus.smithi116 (dependencies changed)... 2024-09-20T19:38:58.986 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:58 smithi116 ceph-mon[23713]: Reconfiguring daemon prometheus.smithi116 on smithi116 2024-09-20T19:38:58.986 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:58 smithi116 ceph-mon[23713]: from='client.? 172.21.15.116:0/4249761690' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:38:59.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:58 smithi118 ceph-mon[25856]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:59.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:58 smithi118 ceph-mon[25856]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:38:59.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:58 smithi118 ceph-mon[25856]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:38:59.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:58 smithi118 ceph-mon[25856]: Reconfiguring prometheus.smithi116 (dependencies changed)... 2024-09-20T19:38:59.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:58 smithi118 ceph-mon[25856]: Reconfiguring daemon prometheus.smithi116 on smithi116 2024-09-20T19:38:59.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:58 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/4249761690' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:38:59.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 108271 11.61 MB/sec execute 109 sec latency 51.409 ms 2024-09-20T19:38:59.991 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:59 smithi116 ceph-mon[23713]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:38:59.991 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:38:59 smithi116 ceph-mon[23713]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-09-20T19:39:00.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:59 smithi118 ceph-mon[25856]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:00.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:38:59 smithi118 ceph-mon[25856]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-09-20T19:39:00.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 108727 11.60 MB/sec execute 110 sec latency 60.677 ms 2024-09-20T19:39:01.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 109040 11.56 MB/sec execute 111 sec latency 58.283 ms 2024-09-20T19:39:02.040 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:01 smithi116 ceph-mon[23713]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-09-20T19:39:02.040 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:01 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:39:02.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:01 smithi118 ceph-mon[25856]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.8 MiB/s wr, 489 op/s 2024-09-20T19:39:02.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:01 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:39:02.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 109435 11.62 MB/sec execute 112 sec latency 63.639 ms 2024-09-20T19:39:03.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 109705 11.58 MB/sec execute 113 sec latency 70.493 ms 2024-09-20T19:39:04.116 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:03 smithi116 ceph-mon[23713]: pgmap v11: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.8 MiB/s wr, 710 op/s 2024-09-20T19:39:04.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:03 smithi118 ceph-mon[25856]: pgmap v11: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.8 MiB/s wr, 710 op/s 2024-09-20T19:39:04.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 110022 11.62 MB/sec execute 114 sec latency 74.231 ms 2024-09-20T19:39:05.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 110349 11.60 MB/sec execute 115 sec latency 57.355 ms 2024-09-20T19:39:05.893 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:05 smithi116 ceph-mon[23713]: pgmap v12: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-20T19:39:06.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:05 smithi118 ceph-mon[25856]: pgmap v12: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-20T19:39:06.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 110570 11.53 MB/sec execute 116 sec latency 51.905 ms 2024-09-20T19:39:07.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 110875 11.47 MB/sec execute 117 sec latency 53.255 ms 2024-09-20T19:39:08.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 111214 11.41 MB/sec execute 118 sec latency 64.659 ms 2024-09-20T19:39:08.355 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:08 smithi116 ceph-mon[23713]: pgmap v13: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-20T19:39:08.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:08 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:08.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:08 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:08.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:08 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:39:08.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:08 smithi118 ceph-mon[25856]: pgmap v13: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-20T19:39:08.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:08 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:08.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:08 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:08.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:08 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:39:09.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 111563 11.40 MB/sec execute 119 sec latency 83.825 ms 2024-09-20T19:39:09.363 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:09 smithi116 ceph-mon[23713]: Failed to get Dashboard config for Prometheus: dashboard get-prometheus-api-host failed: command not known retval: -22 2024-09-20T19:39:09.374 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:09 smithi118 ceph-mon[25856]: Failed to get Dashboard config for Prometheus: dashboard get-prometheus-api-host failed: command not known retval: -22 2024-09-20T19:39:10.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 111981 11.40 MB/sec execute 120 sec latency 39.105 ms 2024-09-20T19:39:10.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:10 smithi116 ceph-mon[23713]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.9 MiB/s wr, 643 op/s 2024-09-20T19:39:10.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:10 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:39:10.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:10 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:39:10.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:10 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:39:10.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:10 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:39:10.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:10 smithi118 ceph-mon[25856]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.9 MiB/s wr, 643 op/s 2024-09-20T19:39:10.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:10 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:39:10.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:10 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T19:39:10.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:10 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T19:39:10.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:10 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:39:11.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 112443 11.36 MB/sec execute 121 sec latency 62.854 ms 2024-09-20T19:39:11.405 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:11 smithi116 ceph-mon[23713]: Upgrade: Updating mgr.smithi118.rmiazv 2024-09-20T19:39:11.405 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:11 smithi116 ceph-mon[23713]: Deploying daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:39:11.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:11 smithi118 ceph-mon[25856]: Upgrade: Updating mgr.smithi118.rmiazv 2024-09-20T19:39:11.598 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:11 smithi118 ceph-mon[25856]: Deploying daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:39:12.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 112896 11.41 MB/sec execute 122 sec latency 32.588 ms 2024-09-20T19:39:12.461 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:12 smithi116 ceph-mon[23713]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 405 op/s 2024-09-20T19:39:12.521 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:12 smithi118 ceph-mon[25856]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 405 op/s 2024-09-20T19:39:13.241 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:13 smithi118 ceph-mon[25856]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 638 op/s 2024-09-20T19:39:13.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 113294 11.44 MB/sec execute 123 sec latency 47.222 ms 2024-09-20T19:39:13.477 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:13 smithi116 ceph-mon[23713]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 638 op/s 2024-09-20T19:39:14.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 113712 11.48 MB/sec execute 124 sec latency 42.648 ms 2024-09-20T19:39:14.512 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:14 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:14.512 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:14 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:14.512 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:14 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:39:14.512 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:14 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:14.512 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:14 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:14.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:14 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:14.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:14 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:14 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:39:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:14 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:14 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:15.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 114040 11.46 MB/sec execute 125 sec latency 41.754 ms 2024-09-20T19:39:15.518 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:15 smithi116 ceph-mon[23713]: pgmap v17: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-09-20T19:39:15.754 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:15 smithi118 ceph-mon[25856]: pgmap v17: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-09-20T19:39:16.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 114467 11.41 MB/sec execute 126 sec latency 54.937 ms 2024-09-20T19:39:17.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 114886 11.37 MB/sec execute 127 sec latency 42.767 ms 2024-09-20T19:39:17.547 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:17 smithi116 ceph-mon[23713]: pgmap v18: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-09-20T19:39:17.547 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:17 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:39:17.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:17 smithi118 ceph-mon[25856]: pgmap v18: 97 pgs: 97 active+clean; 326 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-09-20T19:39:17.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:17 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:39:18.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 115325 11.38 MB/sec execute 128 sec latency 33.589 ms 2024-09-20T19:39:18.819 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:18 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:18.819 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:18 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:18.819 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:18 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:18.820 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:18 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:19.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:18 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:19.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:18 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:19.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:18 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:19.100 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:18 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:19.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 115804 11.38 MB/sec execute 129 sec latency 51.510 ms 2024-09-20T19:39:19.842 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:19 smithi116 ceph-mon[23713]: pgmap v19: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 667 op/s 2024-09-20T19:39:19.842 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:19 smithi116 ceph-mon[23713]: Standby manager daemon smithi118.rmiazv restarted 2024-09-20T19:39:19.842 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:19 smithi116 ceph-mon[23713]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:39:20.181 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:19 smithi118 ceph-mon[25856]: pgmap v19: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 667 op/s 2024-09-20T19:39:20.181 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:19 smithi118 ceph-mon[25856]: Standby manager daemon smithi118.rmiazv restarted 2024-09-20T19:39:20.181 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:19 smithi118 ceph-mon[25856]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:39:20.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 116164 11.34 MB/sec execute 130 sec latency 64.679 ms 2024-09-20T19:39:21.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 116524 11.39 MB/sec execute 131 sec latency 57.441 ms 2024-09-20T19:39:21.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:21 smithi116 ceph-mon[23713]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:39:21.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:21 smithi116 ceph-mon[23713]: mgrmap e35: smithi116.poxcrq(active, since 34s), standbys: smithi118.rmiazv 2024-09-20T19:39:21.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:21 smithi116 ceph-mon[23713]: pgmap v20: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 483 op/s 2024-09-20T19:39:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:21 smithi118 ceph-mon[25856]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:39:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:21 smithi118 ceph-mon[25856]: mgrmap e35: smithi116.poxcrq(active, since 34s), standbys: smithi118.rmiazv 2024-09-20T19:39:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:21 smithi118 ceph-mon[25856]: pgmap v20: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 483 op/s 2024-09-20T19:39:22.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 116962 11.44 MB/sec execute 132 sec latency 59.010 ms 2024-09-20T19:39:23.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.205 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:39:23.205 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:39:23.205 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.205 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:23 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 117390 11.44 MB/sec execute 133 sec latency 56.698 ms 2024-09-20T19:39:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:39:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T19:39:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:23 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:24.225 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: pgmap v21: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-20T19:39:24.225 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:39:24.225 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: Upgrade: Setting container_image for all mgr 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi116.poxcrq"}]: dispatch 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi116.poxcrq"}]: dispatch 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi116.poxcrq"}]': finished 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rmiazv"}]: dispatch 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rmiazv"}]: dispatch 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rmiazv"}]': finished 2024-09-20T19:39:24.226 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:24 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T19:39:24.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 117756 11.41 MB/sec execute 134 sec latency 52.403 ms 2024-09-20T19:39:24.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: pgmap v21: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 719 op/s 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: Upgrade: Setting container_image for all mgr 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi116.poxcrq"}]: dispatch 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi116.poxcrq"}]: dispatch 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi116.poxcrq"}]': finished 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rmiazv"}]: dispatch 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rmiazv"}]: dispatch 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi118.rmiazv"}]': finished 2024-09-20T19:39:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:24 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T19:39:25.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 118180 11.38 MB/sec execute 135 sec latency 32.631 ms 2024-09-20T19:39:26.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 118642 11.37 MB/sec execute 136 sec latency 38.214 ms 2024-09-20T19:39:26.273 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:26 smithi116 ceph-mon[23713]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 486 op/s 2024-09-20T19:39:26.273 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:26 smithi116 ceph-mon[23713]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:26.273 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:26 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T19:39:26.273 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:26 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T19:39:26.273 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:26 smithi116 ceph-mon[23713]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:39:26.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:26 smithi118 ceph-mon[25856]: pgmap v22: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 486 op/s 2024-09-20T19:39:26.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:26 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:26.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:26 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T19:39:26.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:26 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T19:39:26.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:26 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T19:39:27.272 INFO:teuthology.orchestra.run.smithi116.stdout: 5 119152 11.36 MB/sec execute 137 sec latency 40.495 ms 2024-09-20T19:39:27.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:27 smithi116 ceph-mon[23713]: Upgrade: Updating mon.smithi116 2024-09-20T19:39:27.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:27 smithi116 ceph-mon[23713]: Deploying daemon mon.smithi116 on smithi116 2024-09-20T19:39:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:27 smithi118 ceph-mon[25856]: Upgrade: Updating mon.smithi116 2024-09-20T19:39:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:27 smithi118 ceph-mon[25856]: Deploying daemon mon.smithi116 on smithi116 2024-09-20T19:39:28.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 119563 11.36 MB/sec execute 138 sec latency 42.113 ms 2024-09-20T19:39:28.317 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:28 smithi116 ceph-mon[23713]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 486 op/s 2024-09-20T19:39:28.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:28 smithi118 ceph-mon[25856]: pgmap v23: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 486 op/s 2024-09-20T19:39:28.574 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:28 smithi116 systemd[1]: Stopping Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:39:28.825 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:28 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[23708]: 2024-09-20T19:39:28.692+0000 7f2123783700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi116 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:39:28.825 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:28 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[23708]: 2024-09-20T19:39:28.692+0000 7f2123783700 -1 mon.smithi116@0(leader) e2 *** Got Signal Terminated *** 2024-09-20T19:39:28.826 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:28 smithi116 podman[63288]: 2024-09-20 19:39:28.733247208 +0000 UTC m=+0.203461050 container died 3a5bddbb825d1e7ef94008d2eacfb11e06fc138b7f333c0ba117d297da8f8fa3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, ceph=True) 2024-09-20T19:39:29.198 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:39:29.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 119974 11.39 MB/sec execute 139 sec latency 60.610 ms 2024-09-20T19:39:29.856 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:29 smithi116 podman[63288]: 2024-09-20 19:39:29.796082148 +0000 UTC m=+1.266295993 container cleanup 3a5bddbb825d1e7ef94008d2eacfb11e06fc138b7f333c0ba117d297da8f8fa3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, ceph=True, release=754, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True) 2024-09-20T19:39:29.856 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:29 smithi116 bash[63288]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116 2024-09-20T19:39:30.124 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:39:30.124 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (115s) 7s ago 10m 15.5M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:39:30.124 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (10m) 7s ago 10m 6433k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:39:30.124 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (10m) 12s ago 10m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:39:30.124 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (46s) 7s ago 10m 45.0M - dad864ee21e9 8a163a77164a 2024-09-20T19:39:30.124 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (4m) 7s ago 4m 36.2M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:39:30.124 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (4m) 12s ago 4m 13.6M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (70s) 7s ago 12m 486M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (17s) 12s ago 10m 93.4M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (12m) 7s ago 12m 52.2M 2048M 17.2.0 e1d6a67b021e 3a5bddbb825d 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (10m) 12s ago 10m 41.5M 2048M 17.2.0 e1d6a67b021e 849316daa404 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (4m) 7s ago 4m 276M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (4m) 12s ago 4m 84.9M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (105s) 7s ago 10m 9613k - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (79s) 12s ago 9m 11.4M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (9m) 7s ago 9m 442M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (8m) 7s ago 8m 352M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (8m) 7s ago 8m 257M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (7m) 7s ago 7m 405M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (7m) 12s ago 7m 407M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (7m) 12s ago 7m 444M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:39:30.125 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (6m) 12s ago 6m 378M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:39:30.126 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (6m) 12s ago 6m 510M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:39:30.126 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (23s) 7s ago 9m 45.9M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:39:30.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 120572 11.48 MB/sec execute 140 sec latency 25.819 ms 2024-09-20T19:39:30.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:30 smithi116 podman[63299]: 2024-09-20 19:39:30.163478629 +0000 UTC m=+1.431828096 container remove 3a5bddbb825d1e7ef94008d2eacfb11e06fc138b7f333c0ba117d297da8f8fa3 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-20T19:39:30.798 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:39:30.799 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:39:30.800 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:39:30.800 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-20T19:39:30.800 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:39:30.800 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:39:30.800 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:39:30.865 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:30 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service: Deactivated successfully. 2024-09-20T19:39:30.865 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:30 smithi116 systemd[1]: Stopped Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:39:30.865 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:30 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service: Consumed 15.248s CPU time. 2024-09-20T19:39:31.116 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:31 smithi116 systemd[1]: Starting Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:39:31.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 121050 11.50 MB/sec execute 141 sec latency 33.544 ms 2024-09-20T19:39:31.367 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:31 smithi116 podman[63534]: 2024-09-20 19:39:31.199292091 +0000 UTC m=+0.033342341 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:39:31.374 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:39:31.374 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:39:31.374 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:39:31.374 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:39:31.374 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:39:31.375 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr" 2024-09-20T19:39:31.375 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:39:31.375 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "2/23 daemons upgraded", 2024-09-20T19:39:31.375 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T19:39:31.375 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:39:31.375 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:39:31.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:31 smithi116 podman[63534]: 2024-09-20 19:39:31.449608516 +0000 UTC m=+0.283658748 container create f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:39:31.914 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:39:31.914 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:31.914 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:31.915 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:39:31.915 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:39:32.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 121459 11.47 MB/sec execute 142 sec latency 46.135 ms 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 podman[63534]: 2024-09-20 19:39:32.290902699 +0000 UTC m=+1.124952923 container init f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 podman[63534]: 2024-09-20 19:39:32.300051693 +0000 UTC m=+1.134101925 container start f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev), process ceph-mon, pid 2 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: pidfile_write: ignore empty --pid-file 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: load: jerasure load: lrc 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: RocksDB version: 7.9.2 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Git sha 0 2024-09-20T19:39:32.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Compile date 2024-09-20 13:31:19 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: DB SUMMARY 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: DB Session ID: PCRPK0VCAGEIDSO5PM56 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: CURRENT file: CURRENT 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi116/store.db dir, Total Num: 1, files: 000030.sst 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi116/store.db: 000028.log size: 1490767 ; 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.error_if_exists: 0 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.create_if_missing: 0 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.paranoid_checks: 1 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.env: 0x558760a87120 2024-09-20T19:39:32.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.fs: PosixFileSystem 2024-09-20T19:39:32.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.info_log: 0x5587631718e0 2024-09-20T19:39:32.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T19:39:32.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.statistics: (nil) 2024-09-20T19:39:32.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.use_fsync: 0 2024-09-20T19:39:32.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_log_file_size: 0 2024-09-20T19:39:32.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.allow_fallocate: 1 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.use_direct_reads: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.db_log_dir: 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.wal_dir: 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T19:39:32.381 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.write_buffer_manager: 0x558763175b80 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T19:39:32.382 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.unordered_write: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.row_cache: None 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.wal_filter: None 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.two_write_queues: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.wal_compression: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.atomic_flush: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T19:39:32.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.log_readahead_size: 0 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_background_jobs: 2 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_background_compactions: -1 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_subcompactions: 1 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T19:39:32.384 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_open_files: -1 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_background_flushes: -1 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Compression algorithms supported: 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kZSTD supported: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kXpressCompression supported: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kBZip2Compression supported: 0 2024-09-20T19:39:32.385 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kLZ4Compression supported: 1 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kZlibCompression supported: 1 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: kSnappyCompression supported: 1 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi116/store.db/MANIFEST-000015 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.merge_operator: 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_filter: None 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T19:39:32.386 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558763171540) 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: cache_index_and_filter_blocks: 1 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: pin_top_level_index_and_filter: 1 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: index_type: 0 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: data_block_index_type: 0 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: index_shortening: 1 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: checksum: 4 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: no_block_cache: 0 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache: 0x558763165610 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache_name: BinnedLRUCache 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache_options: 2024-09-20T19:39:32.387 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: capacity : 536870912 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: num_shard_bits : 4 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: strict_capacity_limit : 0 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: high_pri_pool_ratio: 0.000 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_cache_compressed: (nil) 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: persistent_cache: (nil) 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_size: 4096 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_size_deviation: 10 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_restart_interval: 16 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: index_block_restart_interval: 1 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: metadata_block_size: 4096 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: partition_filters: 0 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: use_delta_encoding: 1 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: filter_policy: bloomfilter 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: whole_key_filtering: 1 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: verify_compression: 0 2024-09-20T19:39:32.388 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: read_amp_bytes_per_bit: 0 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: format_version: 5 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: enable_index_compression: 1 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: block_align: 0 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: max_auto_readahead_size: 262144 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: prepopulate_block_cache: 0 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: initial_auto_readahead_size: 8192 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: num_file_reads_for_auto_readahead: 2 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression: NoCompression 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T19:39:32.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.num_levels: 7 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T19:39:32.390 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T19:39:32.391 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T19:39:32.392 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T19:39:32.392 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T19:39:32.392 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.arena_block_size: 1048576 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T19:39:32.393 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.inplace_update_support: 0 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T19:39:32.394 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.bloom_locality: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.max_successive_merges: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.ttl: 2592000 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.enable_blob_files: false 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.min_blob_size: 0 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T19:39:32.395 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.blob_file_starting_level: 0 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi116/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11870, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 2e5918d5-f508-48da-99c1-2ef25034ac58 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726861172331290, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726861172336460, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1312450, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11868, "largest_seqno": 12619, "table_properties": {"data_size": 1308424, "index_size": 2154, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 837, "raw_key_size": 7446, "raw_average_key_size": 23, "raw_value_size": 1301366, "raw_average_value_size": 4079, "num_data_blocks": 99, "num_entries": 319, "num_filter_entries": 319, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726861172, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "2e5918d5-f508-48da-99c1-2ef25034ac58", "db_session_id": "PCRPK0VCAGEIDSO5PM56", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726861172336581, "job": 1, "event": "recovery_finished"} 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-20T19:39:32.396 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi116/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558763194e00 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: DB pointer 0x558763296000 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** DB Stats ** 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** Compaction Stats [default] ** 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: L0 1/0 1.25 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 435.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: L6 1/0 7.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Sum 2/0 8.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 435.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 435.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** Compaction Stats [default] ** 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 435.1 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Cumulative compaction: 0.00 GB write, 160.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:39:32.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Interval compaction: 0.00 GB write, 160.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Block cache BinnedLRUCache@0x558763165610#2 capacity: 512.00 MB usage: 3.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,2.33 KB,0.000444055%) Misc(1,0.00 KB,0%) 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: starting mon.smithi116 rank 0 at public addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] at bind addrs [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi116 fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???) e2 preinit fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???).mds e6 new map 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???).mds e6 print_map 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: e6 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: legacy client fscid: 1 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Filesystem 'foofs' (1) 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: fs_name foofs 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: epoch 5 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: created 2024-09-20T19:34:35.213481+0000 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: modified 2024-09-20T19:34:41.252392+0000 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: tableserver 0 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: root 0 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: session_timeout 60 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: session_autoclose 300 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: max_file_size 1099511627776 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: max_xattr_size 65536 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: required_client_features {} 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: last_failure 0 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: last_failure_osd_epoch 0 2024-09-20T19:39:32.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: max_mds 1 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: in 0 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: up {0=14494} 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: failed 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: damaged 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: stopped 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: data_pools [3] 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: metadata_pool 2 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: inline_data disabled 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: balancer 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: bal_rank_mask -1 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: standby_count_wanted 1 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: qdb_cluster leader: 0 members: 2024-09-20T19:39:32.402 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: [mds.foofs.smithi116.xvfgaf{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.116:6834/1570267385,v1:172.21.15.116:6835/1570267385] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Standby daemons: 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: [mds.foofs.smithi118.zumcyi{-1:24271} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T19:39:32.403 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 ceph-mon[63578]: mon.smithi116@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-20T19:39:32.629 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 bash[63534]: f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 2024-09-20T19:39:32.629 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:32 smithi116 systemd[1]: Started Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:39:33.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 121807 11.42 MB/sec execute 143 sec latency 41.128 ms 2024-09-20T19:39:34.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 121905 11.36 MB/sec execute 144 sec latency 1030.751 ms 2024-09-20T19:39:35.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 121905 11.28 MB/sec execute 145 sec latency 2030.804 ms 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 756 op/s 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='client.24549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='client.24557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 505 op/s 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/4052007816' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='client.24565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2275282332' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 781 op/s 2024-09-20T19:39:35.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 545 op/s 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: mon.smithi116 calling monitor election 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: mon.smithi116 is new leader, mons smithi116,smithi118 in quorum (ranks 0,1) 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: monmap epoch 2 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: last_changed 2024-09-20T19:29:23.022718+0000 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: created 2024-09-20T19:27:15.298011+0000 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: min_mon_release 17 (quincy) 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: election_strategy: 1 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: 0: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.smithi116 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: mgrmap e35: smithi116.poxcrq(active, since 50s), standbys: smithi118.rmiazv 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: Health detail: HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:39:35.617 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: [WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:35.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:35.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: [WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:39:35.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:39:35.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: from='mgr.14762 ' entity='' 2024-09-20T19:39:35.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:35 smithi116 ceph-mon[63578]: mgrmap e36: smithi116.poxcrq(active, since 50s), standbys: smithi118.rmiazv 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 756 op/s 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='client.24549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='client.24557 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 505 op/s 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/4052007816' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='client.24565 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='mgr.14762 172.21.15.116:0/246493537' entity='mgr.smithi116.poxcrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='client.? 172.21.15.116:0/2275282332' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T19:39:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 781 op/s 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 545 op/s 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: mon.smithi116 calling monitor election 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: mon.smithi116 is new leader, mons smithi116,smithi118 in quorum (ranks 0,1) 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: monmap epoch 2 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: last_changed 2024-09-20T19:29:23.022718+0000 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: created 2024-09-20T19:27:15.298011+0000 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: min_mon_release 17 (quincy) 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: election_strategy: 1 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: 0: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.smithi116 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: mgrmap e35: smithi116.poxcrq(active, since 50s), standbys: smithi118.rmiazv 2024-09-20T19:39:35.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: Health detail: HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:39:35.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: [WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:35.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:35.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: [WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:39:35.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:39:35.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: from='mgr.14762 ' entity='' 2024-09-20T19:39:35.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:35 smithi118 ceph-mon[25856]: mgrmap e36: smithi116.poxcrq(active, since 50s), standbys: smithi118.rmiazv 2024-09-20T19:39:36.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 122477 11.31 MB/sec execute 146 sec latency 2121.525 ms 2024-09-20T19:39:37.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 123268 11.35 MB/sec execute 147 sec latency 12.161 ms 2024-09-20T19:39:37.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:37 smithi116 ceph-mon[63578]: mgrmap e37: smithi116.poxcrq(active, since 51s), standbys: smithi118.rmiazv 2024-09-20T19:39:37.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:37 smithi118 ceph-mon[25856]: mgrmap e37: smithi116.poxcrq(active, since 51s), standbys: smithi118.rmiazv 2024-09-20T19:39:38.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 123700 11.39 MB/sec execute 148 sec latency 40.745 ms 2024-09-20T19:39:39.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 124106 11.41 MB/sec execute 149 sec latency 66.224 ms 2024-09-20T19:39:40.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 124381 11.40 MB/sec execute 150 sec latency 59.067 ms 2024-09-20T19:39:41.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 124596 11.37 MB/sec execute 151 sec latency 48.709 ms 2024-09-20T19:39:42.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:41 smithi116 ceph-mon[63578]: Standby manager daemon smithi118.rmiazv restarted 2024-09-20T19:39:42.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:41 smithi116 ceph-mon[63578]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:39:42.273 INFO:teuthology.orchestra.run.smithi116.stdout: 5 124809 11.33 MB/sec execute 152 sec latency 62.089 ms 2024-09-20T19:39:42.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: Standby manager daemon smithi118.rmiazv restarted 2024-09-20T19:39:42.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: Standby manager daemon smithi118.rmiazv started 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: mgrmap e38: smithi116.poxcrq(active, since 56s), standbys: smithi118.rmiazv 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: Active manager daemon smithi116.poxcrq restarted 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: Activating manager daemon smithi116.poxcrq 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: mgrmap e39: smithi116.poxcrq(active, starting, since 0.00691359s), standbys: smithi118.rmiazv 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi116"} : dispatch 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi118"} : dispatch 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"} : dispatch 2024-09-20T19:39:43.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr metadata", "who": "smithi118.rmiazv", "id": "smithi118.rmiazv"} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata"} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata"} : dispatch 2024-09-20T19:39:43.086 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata"} : dispatch 2024-09-20T19:39:43.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:39:43.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:39:43.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:39:43.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-20T19:39:43.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"} : dispatch 2024-09-20T19:39:43.087 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"} : dispatch 2024-09-20T19:39:43.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 125071 11.27 MB/sec execute 153 sec latency 142.969 ms 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: Failed to load ceph-mgr modules: dashboard 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: mgrmap e38: smithi116.poxcrq(active, since 56s), standbys: smithi118.rmiazv 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: Active manager daemon smithi116.poxcrq restarted 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: Activating manager daemon smithi116.poxcrq 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: mgrmap e39: smithi116.poxcrq(active, starting, since 0.00691359s), standbys: smithi118.rmiazv 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi116"} : dispatch 2024-09-20T19:39:43.431 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi118"} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr metadata", "who": "smithi116.poxcrq", "id": "smithi116.poxcrq"} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr metadata", "who": "smithi118.rmiazv", "id": "smithi118.rmiazv"} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata"} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata"} : dispatch 2024-09-20T19:39:43.432 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata"} : dispatch 2024-09-20T19:39:43.433 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: Manager daemon smithi116.poxcrq is now available 2024-09-20T19:39:43.433 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:39:43.433 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:39:43.433 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-20T19:39:43.433 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/mirror_snapshot_schedule"} : dispatch 2024-09-20T19:39:43.433 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:42 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi116.poxcrq/trash_purge_schedule"} : dispatch 2024-09-20T19:39:44.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 125303 11.21 MB/sec execute 154 sec latency 52.336 ms 2024-09-20T19:39:44.369 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:44 smithi116 ceph-mon[63578]: mgrmap e40: smithi116.poxcrq(active, since 1.02561s), standbys: smithi118.rmiazv 2024-09-20T19:39:44.369 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:44 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T19:39:44.369 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:44 smithi116 ceph-mon[63578]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-20T19:39:44.415 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:44 smithi118 ceph-mon[25856]: mgrmap e40: smithi116.poxcrq(active, since 1.02561s), standbys: smithi118.rmiazv 2024-09-20T19:39:44.415 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:44 smithi118 ceph-mon[25856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T19:39:44.415 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:44 smithi118 ceph-mon[25856]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-20T19:39:45.140 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:45 smithi116 ceph-mon[63578]: mgrmap e41: smithi116.poxcrq(active, since 2s), standbys: smithi118.rmiazv 2024-09-20T19:39:45.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 125544 11.17 MB/sec execute 155 sec latency 74.462 ms 2024-09-20T19:39:45.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:45 smithi118 ceph-mon[25856]: mgrmap e41: smithi116.poxcrq(active, since 2s), standbys: smithi118.rmiazv 2024-09-20T19:39:46.155 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:46 smithi116 ceph-mon[63578]: [20/Sep/2024:19:39:44] ENGINE Bus STARTING 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:46 smithi116 ceph-mon[63578]: [20/Sep/2024:19:39:44] ENGINE Serving on http://172.21.15.116:8765 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:46 smithi116 ceph-mon[63578]: [20/Sep/2024:19:39:44] ENGINE Serving on https://172.21.15.116:7150 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:46 smithi116 ceph-mon[63578]: [20/Sep/2024:19:39:44] ENGINE Bus STARTED 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:46 smithi116 ceph-mon[63578]: [20/Sep/2024:19:39:44] ENGINE Error in HTTPServer.serve 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: Traceback (most recent call last): 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._connections.run(self.expiration_interval) 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._run(expiration_interval) 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T19:39:46.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: s = self.context.wrap_socket( 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: return self.sslsocket_class._create( 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self.do_handshake() 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: self._sslobj.do_handshake() 2024-09-20T19:39:46.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T19:39:46.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 125903 11.19 MB/sec execute 156 sec latency 67.974 ms 2024-09-20T19:39:46.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:46 smithi118 ceph-mon[25856]: [20/Sep/2024:19:39:44] ENGINE Bus STARTING 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:46 smithi118 ceph-mon[25856]: [20/Sep/2024:19:39:44] ENGINE Serving on http://172.21.15.116:8765 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:46 smithi118 ceph-mon[25856]: [20/Sep/2024:19:39:44] ENGINE Serving on https://172.21.15.116:7150 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:46 smithi118 ceph-mon[25856]: [20/Sep/2024:19:39:44] ENGINE Bus STARTED 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:46 smithi118 ceph-mon[25856]: [20/Sep/2024:19:39:44] ENGINE Error in HTTPServer.serve 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Traceback (most recent call last): 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._connections.run(self.expiration_interval) 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._run(expiration_interval) 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T19:39:46.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: s = self.context.wrap_socket( 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: return self.sslsocket_class._create( 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self.do_handshake() 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: self._sslobj.do_handshake() 2024-09-20T19:39:46.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T19:39:47.187 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:47 smithi116 ceph-mon[63578]: mgrmap e42: smithi116.poxcrq(active, since 4s), standbys: smithi118.rmiazv 2024-09-20T19:39:47.187 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:47 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:47.187 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:47 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:47.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 126296 11.15 MB/sec execute 157 sec latency 56.578 ms 2024-09-20T19:39:47.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:47 smithi118 ceph-mon[25856]: mgrmap e42: smithi116.poxcrq(active, since 4s), standbys: smithi118.rmiazv 2024-09-20T19:39:47.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:47 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:47.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:47 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:48.222 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:48 smithi116 ceph-mon[63578]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-20T19:39:48.222 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:48 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:48.222 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:48 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:48.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 126611 11.13 MB/sec execute 158 sec latency 54.763 ms 2024-09-20T19:39:48.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:48 smithi118 ceph-mon[25856]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-20T19:39:48.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:48 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:48.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:48 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:49.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 126981 11.13 MB/sec execute 159 sec latency 41.420 ms 2024-09-20T19:39:50.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:50 smithi118 ceph-mon[25856]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 41 KiB/s rd, 3.8 MiB/s wr, 448 op/s 2024-09-20T19:39:50.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:50 smithi116 ceph-mon[63578]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 41 KiB/s rd, 3.8 MiB/s wr, 448 op/s 2024-09-20T19:39:50.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 127370 11.15 MB/sec execute 160 sec latency 35.716 ms 2024-09-20T19:39:51.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 127827 11.20 MB/sec execute 161 sec latency 31.449 ms 2024-09-20T19:39:52.047 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: pgmap v7: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 325 op/s 2024-09-20T19:39:52.047 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.048 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.048 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.048 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.048 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.048 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.048 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:39:52.048 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: pgmap v7: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 325 op/s 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:39:52.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:51 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:39:52.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 128071 11.17 MB/sec execute 162 sec latency 63.732 ms 2024-09-20T19:39:53.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 128318 11.15 MB/sec execute 163 sec latency 84.142 ms 2024-09-20T19:39:53.327 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:39:53.327 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:39:53.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:39:53.329 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "quorum_status"} : dispatch 2024-09-20T19:39:53.369 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi116:/etc/ceph/ceph.conf 2024-09-20T19:39:53.369 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.conf 2024-09-20T19:39:53.369 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:39:53.369 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi118:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi116:/etc/ceph/ceph.client.admin.keyring 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: Updating smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:39:53.370 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:53 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "quorum_status"} : dispatch 2024-09-20T19:39:54.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 128594 11.09 MB/sec execute 164 sec latency 94.257 ms 2024-09-20T19:39:54.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:54 smithi118 ceph-mon[25856]: pgmap v8: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 255 op/s 2024-09-20T19:39:54.595 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:54 smithi116 ceph-mon[63578]: pgmap v8: 97 pgs: 97 active+clean; 326 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 255 op/s 2024-09-20T19:39:55.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 128821 11.04 MB/sec execute 165 sec latency 139.486 ms 2024-09-20T19:39:55.650 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:55 smithi116 ceph-mon[63578]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.4 MiB/s wr, 434 op/s 2024-09-20T19:39:55.650 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:55 smithi116 ceph-mon[63578]: Upgrade: Updating mon.smithi118 2024-09-20T19:39:55.650 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:55.651 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T19:39:55.651 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T19:39:55.651 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:39:55.651 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:55 smithi116 ceph-mon[63578]: Deploying daemon mon.smithi118 on smithi118 2024-09-20T19:39:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:55 smithi118 ceph-mon[25856]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.4 MiB/s wr, 434 op/s 2024-09-20T19:39:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:55 smithi118 ceph-mon[25856]: Upgrade: Updating mon.smithi118 2024-09-20T19:39:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:55 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:55 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T19:39:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:55 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T19:39:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:55 smithi118 ceph-mon[25856]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:39:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:55 smithi118 ceph-mon[25856]: Deploying daemon mon.smithi118 on smithi118 2024-09-20T19:39:56.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 129088 11.01 MB/sec execute 166 sec latency 39.877 ms 2024-09-20T19:39:56.883 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:56 smithi118 systemd[1]: Stopping Ceph mon.smithi118 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:39:57.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:56 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118[25803]: 2024-09-20T19:39:56.880+0000 7f6dd1bda700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi118 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:39:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:56 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118[25803]: 2024-09-20T19:39:56.880+0000 7f6dd1bda700 -1 mon.smithi118@1(peon) e2 *** Got Signal Terminated *** 2024-09-20T19:39:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:57 smithi118 podman[49036]: 2024-09-20 19:39:57.104963401 +0000 UTC m=+0.342461846 container died 849316daa404d4446dce5e8d6973a1e0c81afe0ebe76b4caf8f8d87e9c59d2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, GIT_BRANCH=HEAD, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, ceph=True, architecture=x86_64, release=754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-20T19:39:57.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 129444 11.02 MB/sec execute 167 sec latency 65.528 ms 2024-09-20T19:39:57.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:57 smithi118 podman[49036]: 2024-09-20 19:39:57.499522318 +0000 UTC m=+0.737020758 container cleanup 849316daa404d4446dce5e8d6973a1e0c81afe0ebe76b4caf8f8d87e9c59d2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, distribution-scope=public, io.openshift.expose-services=, RELEASE=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, maintainer=Guillaume Abrioux ) 2024-09-20T19:39:57.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:57 smithi118 bash[49036]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118 2024-09-20T19:39:58.168 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:57 smithi118 podman[49047]: 2024-09-20 19:39:57.801903185 +0000 UTC m=+0.691328779 container remove 849316daa404d4446dce5e8d6973a1e0c81afe0ebe76b4caf8f8d87e9c59d2ab (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , RELEASE=HEAD, release=754, vendor=Red Hat, Inc., vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, name=centos-stream, io.openshift.expose-services=, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, architecture=x86_64) 2024-09-20T19:39:58.168 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:57 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi118.service: Deactivated successfully. 2024-09-20T19:39:58.168 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:57 smithi118 systemd[1]: Stopped Ceph mon.smithi118 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:39:58.168 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:57 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi118.service: Consumed 6.842s CPU time. 2024-09-20T19:39:58.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 129840 10.99 MB/sec execute 168 sec latency 43.602 ms 2024-09-20T19:39:58.420 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 systemd[1]: Starting Ceph mon.smithi118 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:39:58.420 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 podman[49178]: 2024-09-20 19:39:58.303941639 +0000 UTC m=+0.020994691 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:39:58.420 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 podman[49178]: 2024-09-20 19:39:58.419833883 +0000 UTC m=+0.136886942 container create 7103ca9965a41f267220b67dcddbf3350f6496ec91d5203110baccf1fd501ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True) 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 podman[49178]: 2024-09-20 19:39:58.698255582 +0000 UTC m=+0.415308643 container init 7103ca9965a41f267220b67dcddbf3350f6496ec91d5203110baccf1fd501ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2) 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 podman[49178]: 2024-09-20 19:39:58.701686445 +0000 UTC m=+0.418739500 container start 7103ca9965a41f267220b67dcddbf3350f6496ec91d5203110baccf1fd501ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev), process ceph-mon, pid 2 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: pidfile_write: ignore empty --pid-file 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: load: jerasure load: lrc 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: RocksDB version: 7.9.2 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Git sha 0 2024-09-20T19:39:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Compile date 2024-09-20 13:31:19 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: DB SUMMARY 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: DB Session ID: 7LICILMSWOLPAYDALVVM 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: CURRENT file: CURRENT 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi118/store.db dir, Total Num: 1, files: 000024.sst 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi118/store.db: 000022.log size: 5417243 ; 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.error_if_exists: 0 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.create_if_missing: 0 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.paranoid_checks: 1 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T19:39:58.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-20T19:39:58.967 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.env: 0x55bffbec6120 2024-09-20T19:39:58.967 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.fs: PosixFileSystem 2024-09-20T19:39:58.967 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.info_log: 0x55bffca458e0 2024-09-20T19:39:58.967 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T19:39:58.968 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.statistics: (nil) 2024-09-20T19:39:58.968 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.use_fsync: 0 2024-09-20T19:39:58.968 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_log_file_size: 0 2024-09-20T19:39:58.969 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T19:39:58.969 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T19:39:58.969 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T19:39:58.969 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T19:39:58.969 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.allow_fallocate: 1 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.use_direct_reads: 0 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.db_log_dir: 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.wal_dir: 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T19:39:58.970 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.write_buffer_manager: 0x55bffca49b80 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.unordered_write: 0 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T19:39:58.971 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.row_cache: None 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.wal_filter: None 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.two_write_queues: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.wal_compression: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.atomic_flush: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T19:39:58.972 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.log_readahead_size: 0 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_background_jobs: 2 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_background_compactions: -1 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_subcompactions: 1 2024-09-20T19:39:58.973 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_open_files: -1 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T19:39:58.974 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_background_flushes: -1 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Compression algorithms supported: 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kZSTD supported: 0 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kXpressCompression supported: 0 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kBZip2Compression supported: 0 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kLZ4Compression supported: 1 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kZlibCompression supported: 1 2024-09-20T19:39:58.975 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T19:39:58.976 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: kSnappyCompression supported: 1 2024-09-20T19:39:58.976 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T19:39:58.976 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-20T19:39:58.976 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi118/store.db/MANIFEST-000009 2024-09-20T19:39:58.976 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-20T19:39:58.976 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T19:39:58.976 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.merge_operator: 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_filter: None 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bffca45540) 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: cache_index_and_filter_blocks: 1 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: pin_top_level_index_and_filter: 1 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_type: 0 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_block_index_type: 0 2024-09-20T19:39:58.977 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_shortening: 1 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: checksum: 4 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: no_block_cache: 0 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache: 0x55bffca39610 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_name: BinnedLRUCache 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_options: 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: capacity : 536870912 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: num_shard_bits : 4 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: strict_capacity_limit : 0 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: high_pri_pool_ratio: 0.000 2024-09-20T19:39:58.978 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_cache_compressed: (nil) 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: persistent_cache: (nil) 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_size: 4096 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_size_deviation: 10 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_restart_interval: 16 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: index_block_restart_interval: 1 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: metadata_block_size: 4096 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: partition_filters: 0 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: use_delta_encoding: 1 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: filter_policy: bloomfilter 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: whole_key_filtering: 1 2024-09-20T19:39:58.979 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: verify_compression: 0 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: read_amp_bytes_per_bit: 0 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: format_version: 5 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: enable_index_compression: 1 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: block_align: 0 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_auto_readahead_size: 262144 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: prepopulate_block_cache: 0 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: initial_auto_readahead_size: 8192 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: num_file_reads_for_auto_readahead: 2 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T19:39:58.980 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression: NoCompression 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.num_levels: 7 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T19:39:58.981 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T19:39:58.982 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T19:39:58.983 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.arena_block_size: 1048576 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T19:39:58.984 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.inplace_update_support: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.bloom_locality: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.max_successive_merges: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T19:39:58.985 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.ttl: 2592000 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.enable_blob_files: false 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.min_blob_size: 0 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.blob_file_starting_level: 0 2024-09-20T19:39:58.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi118/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12277, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dc45db36-37d6-46b5-88be-5d6644f1fe04 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726861198733788, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726861198749234, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3315135, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12282, "largest_seqno": 13632, "table_properties": {"data_size": 3308919, "index_size": 3763, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 12769, "raw_average_key_size": 23, "raw_value_size": 3296979, "raw_average_value_size": 6049, "num_data_blocks": 172, "num_entries": 545, "num_filter_entries": 545, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726861198, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dc45db36-37d6-46b5-88be-5d6644f1fe04", "db_session_id": "7LICILMSWOLPAYDALVVM", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726861198749388, "job": 1, "event": "recovery_finished"} 2024-09-20T19:39:58.987 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi118/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bffca68e00 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: DB pointer 0x55bffcb6a000 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** DB Stats ** 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T19:39:58.988 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: L0 1/0 3.16 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 498.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: L6 1/0 7.19 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Sum 2/0 10.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 498.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 498.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** Compaction Stats [default] ** 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T19:39:58.989 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 498.3 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T19:39:58.990 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Cumulative compaction: 0.00 GB write, 164.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Interval compaction: 0.00 GB write, 164.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Block cache BinnedLRUCache@0x55bffca39610#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: starting mon.smithi118 rank 1 at public addrs [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] at bind addrs [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi118 fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???) e2 preinit fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???).mds e6 new map 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???).mds e6 print_map 2024-09-20T19:39:58.991 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: e6 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: legacy client fscid: 1 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Filesystem 'foofs' (1) 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: fs_name foofs 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: epoch 5 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: created 2024-09-20T19:34:35.213481+0000 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: modified 2024-09-20T19:34:41.252392+0000 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: tableserver 0 2024-09-20T19:39:58.992 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: root 0 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: session_timeout 60 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: session_autoclose 300 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_file_size 1099511627776 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_xattr_size 65536 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: required_client_features {} 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: last_failure 0 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: last_failure_osd_epoch 0 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: max_mds 1 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: in 0 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: up {0=14494} 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: failed 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: damaged 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: stopped 2024-09-20T19:39:58.993 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: data_pools [3] 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: metadata_pool 2 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: inline_data disabled 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: balancer 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: bal_rank_mask -1 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: standby_count_wanted 1 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: qdb_cluster leader: 0 members: 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: [mds.foofs.smithi116.xvfgaf{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.116:6834/1570267385,v1:172.21.15.116:6835/1570267385] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: Standby daemons: 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout: [mds.foofs.smithi118.zumcyi{-1:24271} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-20T19:39:58.994 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: mon.smithi118@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 bash[49178]: 7103ca9965a41f267220b67dcddbf3350f6496ec91d5203110baccf1fd501ea8 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:58 smithi118 systemd[1]: Started Ceph mon.smithi118 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:39:58.995 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:58 smithi116 ceph-mon[63578]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-20T19:39:58.996 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:39:59.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 130220 10.99 MB/sec execute 169 sec latency 39.414 ms 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi116"} : dispatch 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: mon.smithi116 calling monitor election 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: mon.smithi118 calling monitor election 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: mon.smithi116 is new leader, mons smithi116,smithi118 in quorum (ranks 0,1) 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi118"} : dispatch 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: monmap epoch 3 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: last_changed 2024-09-20T19:39:58.799477+0000 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: created 2024-09-20T19:27:15.298011+0000 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: min_mon_release 19 (squid) 2024-09-20T19:39:59.997 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: election_strategy: 1 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: 0: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.smithi116 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: mgrmap e42: smithi116.poxcrq(active, since 16s), standbys: smithi118.rmiazv 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: Health detail: HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: [WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: [WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:39:59.998 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:39:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi116"} : dispatch 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: mon.smithi116 calling monitor election 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: mon.smithi118 calling monitor election 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: mon.smithi116 is new leader, mons smithi116,smithi118 in quorum (ranks 0,1) 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mon metadata", "id": "smithi118"} : dispatch 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: monmap epoch 3 2024-09-20T19:40:00.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: fsid 2d2015e0-7786-11ef-baf5-efdc52797490 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: last_changed 2024-09-20T19:39:58.799477+0000 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: created 2024-09-20T19:27:15.298011+0000 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: min_mon_release 19 (squid) 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: election_strategy: 1 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: 0: [v2:172.21.15.116:3300/0,v1:172.21.15.116:6789/0] mon.smithi116 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: 1: [v2:172.21.15.118:3300/0,v1:172.21.15.118:6789/0] mon.smithi118 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: mgrmap e42: smithi116.poxcrq(active, since 16s), standbys: smithi118.rmiazv 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: Health detail: HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: [WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: [WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:40:00.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:00.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:00.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:39:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:40:00.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 130560 10.98 MB/sec execute 170 sec latency 67.146 ms 2024-09-20T19:40:00.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:00 smithi118 ceph-mon[49197]: overall HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:40:01.015 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:00 smithi116 ceph-mon[63578]: overall HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:40:01.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 130944 11.00 MB/sec execute 171 sec latency 47.662 ms 2024-09-20T19:40:02.034 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:01 smithi116 ceph-mon[63578]: pgmap v12: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-20T19:40:02.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:01 smithi118 ceph-mon[49197]: pgmap v12: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-20T19:40:02.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 131352 11.04 MB/sec execute 172 sec latency 56.252 ms 2024-09-20T19:40:02.362 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:40:02.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:02 smithi118 ceph-mon[49197]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:03.072 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:02 smithi116 ceph-mon[63578]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:03.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 131571 11.01 MB/sec execute 173 sec latency 58.808 ms 2024-09-20T19:40:03.303 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (2m) 12s ago 11m 15.6M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (11m) 12s ago 11m 6433k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (10m) 0s ago 10m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (79s) 12s ago 11m 43.5M - dad864ee21e9 8a163a77164a 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (5m) 12s ago 5m 36.9M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (5m) 0s ago 5m 14.0M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (103s) 12s ago 12m 468M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (50s) 0s ago 10m 446M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (32s) 12s ago 12m 32.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (4s) 0s ago 10m 25.1M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (5m) 12s ago 5m 280M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (5m) 0s ago 5m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:40:03.304 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (2m) 12s ago 11m 9726k - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (112s) 0s ago 10m 11.3M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (9m) 12s ago 9m 461M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (9m) 12s ago 9m 398M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (8m) 12s ago 8m 271M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (8m) 12s ago 8m 425M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (8m) 0s ago 8m 480M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (7m) 0s ago 7m 512M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (7m) 0s ago 7m 430M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (6m) 0s ago 6m 576M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:40:03.305 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (56s) 12s ago 10m 46.2M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:40:03.787 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:40:03.787 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:40:03.787 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:40:03.787 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:03.787 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 4 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:40:03.788 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr" 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "4/23 daemons upgraded", 2024-09-20T19:40:04.170 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T19:40:04.171 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:40:04.171 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:40:04.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 131773 10.97 MB/sec execute 174 sec latency 63.834 ms 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: pgmap v13: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1362781330' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: pgmap v13: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-20T19:40:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1362781330' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:04.632 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:40:04.632 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:40:04.632 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:40:04.632 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:40:04.632 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:40:05.133 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:05 smithi116 ceph-mon[63578]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:05.133 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:05 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1882164570' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:40:05.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 132022 10.94 MB/sec execute 175 sec latency 116.096 ms 2024-09-20T19:40:05.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:05 smithi118 ceph-mon[49197]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:05.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:05 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1882164570' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:40:06.137 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:06 smithi116 ceph-mon[63578]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-09-20T19:40:06.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 132328 10.89 MB/sec execute 176 sec latency 75.763 ms 2024-09-20T19:40:06.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:06 smithi118 ceph-mon[49197]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.4 MiB/s wr, 604 op/s 2024-09-20T19:40:07.274 INFO:teuthology.orchestra.run.smithi116.stdout: 5 132602 10.86 MB/sec execute 177 sec latency 63.332 ms 2024-09-20T19:40:08.197 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-20T19:40:08.197 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T19:40:08.198 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:08.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-20T19:40:08.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T19:40:08.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:08.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 132985 10.89 MB/sec execute 178 sec latency 59.412 ms 2024-09-20T19:40:09.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:09 smithi116 ceph-mon[63578]: Reconfiguring mon.smithi116 (monmap changed)... 2024-09-20T19:40:09.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:09 smithi116 ceph-mon[63578]: Reconfiguring daemon mon.smithi116 on smithi116 2024-09-20T19:40:09.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 133398 10.85 MB/sec execute 179 sec latency 45.651 ms 2024-09-20T19:40:09.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:09 smithi118 ceph-mon[49197]: Reconfiguring mon.smithi116 (monmap changed)... 2024-09-20T19:40:09.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:09 smithi118 ceph-mon[49197]: Reconfiguring daemon mon.smithi116 on smithi116 2024-09-20T19:40:10.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 133744 10.85 MB/sec execute 180 sec latency 42.635 ms 2024-09-20T19:40:10.303 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:10 smithi116 ceph-mon[63578]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.6 MiB/s wr, 578 op/s 2024-09-20T19:40:10.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:10 smithi118 ceph-mon[49197]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.6 MiB/s wr, 578 op/s 2024-09-20T19:40:11.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 134006 10.81 MB/sec execute 181 sec latency 36.115 ms 2024-09-20T19:40:11.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:11.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:11.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:11 smithi116 ceph-mon[63578]: Reconfiguring mgr.smithi116.poxcrq (monmap changed)... 2024-09-20T19:40:11.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T19:40:11.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T19:40:11.368 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:11.369 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:11 smithi116 ceph-mon[63578]: Reconfiguring daemon mgr.smithi116.poxcrq on smithi116 2024-09-20T19:40:11.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:11 smithi118 ceph-mon[49197]: Reconfiguring mgr.smithi116.poxcrq (monmap changed)... 2024-09-20T19:40:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi116.poxcrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T19:40:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T19:40:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:11 smithi118 ceph-mon[49197]: Reconfiguring daemon mgr.smithi116.poxcrq on smithi116 2024-09-20T19:40:12.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 134340 10.84 MB/sec execute 182 sec latency 57.094 ms 2024-09-20T19:40:12.401 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:12 smithi116 ceph-mon[63578]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-09-20T19:40:12.615 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:12 smithi118 ceph-mon[49197]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-09-20T19:40:13.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 134718 10.85 MB/sec execute 183 sec latency 57.290 ms 2024-09-20T19:40:13.411 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:13.411 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:13.411 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:13.412 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:13.412 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:13.626 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:13.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:13.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:13.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:13.627 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:14.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 135053 10.86 MB/sec execute 184 sec latency 38.526 ms 2024-09-20T19:40:14.423 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:14 smithi116 ceph-mon[63578]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-09-20T19:40:14.423 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:14 smithi116 ceph-mon[63578]: Reconfiguring crash.smithi116 (monmap changed)... 2024-09-20T19:40:14.423 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:14 smithi116 ceph-mon[63578]: Reconfiguring daemon crash.smithi116 on smithi116 2024-09-20T19:40:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:14 smithi118 ceph-mon[49197]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-09-20T19:40:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:14 smithi118 ceph-mon[49197]: Reconfiguring crash.smithi116 (monmap changed)... 2024-09-20T19:40:14.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:14 smithi118 ceph-mon[49197]: Reconfiguring daemon crash.smithi116 on smithi116 2024-09-20T19:40:15.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 135281 10.84 MB/sec execute 185 sec latency 73.845 ms 2024-09-20T19:40:16.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:16 smithi116 ceph-mon[63578]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.2 MiB/s wr, 587 op/s 2024-09-20T19:40:16.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:16.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:16.215 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:16 smithi116 ceph-mon[63578]: Reconfiguring osd.0 (monmap changed)... 2024-09-20T19:40:16.216 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T19:40:16.216 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:16.216 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:16 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.0 on smithi116 2024-09-20T19:40:16.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 135592 10.81 MB/sec execute 186 sec latency 84.635 ms 2024-09-20T19:40:16.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:16 smithi118 ceph-mon[49197]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.2 MiB/s wr, 587 op/s 2024-09-20T19:40:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:16 smithi118 ceph-mon[49197]: Reconfiguring osd.0 (monmap changed)... 2024-09-20T19:40:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T19:40:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:16 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.0 on smithi116 2024-09-20T19:40:17.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 135976 10.78 MB/sec execute 187 sec latency 93.813 ms 2024-09-20T19:40:18.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:18 smithi116 ceph-mon[63578]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 377 op/s 2024-09-20T19:40:18.259 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:18.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:18.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:18 smithi116 ceph-mon[63578]: Reconfiguring osd.1 (monmap changed)... 2024-09-20T19:40:18.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T19:40:18.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:18.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:18 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.1 on smithi116 2024-09-20T19:40:18.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 136455 10.80 MB/sec execute 188 sec latency 41.914 ms 2024-09-20T19:40:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:18 smithi118 ceph-mon[49197]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 377 op/s 2024-09-20T19:40:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:18 smithi118 ceph-mon[49197]: Reconfiguring osd.1 (monmap changed)... 2024-09-20T19:40:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T19:40:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:18 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.1 on smithi116 2024-09-20T19:40:19.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 136902 10.78 MB/sec execute 189 sec latency 64.092 ms 2024-09-20T19:40:20.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 137325 10.78 MB/sec execute 190 sec latency 65.984 ms 2024-09-20T19:40:20.277 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:20 smithi116 ceph-mon[63578]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.9 MiB/s wr, 594 op/s 2024-09-20T19:40:20.277 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:20 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:20.277 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:20 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:20.278 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:20 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T19:40:20.278 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:20 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:20.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:20 smithi118 ceph-mon[49197]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.9 MiB/s wr, 594 op/s 2024-09-20T19:40:20.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:20 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:20.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:20 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:20.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:20 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T19:40:20.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:20 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:21.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 137707 10.79 MB/sec execute 191 sec latency 65.455 ms 2024-09-20T19:40:21.305 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:21 smithi116 ceph-mon[63578]: Reconfiguring osd.2 (monmap changed)... 2024-09-20T19:40:21.306 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:21 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.2 on smithi116 2024-09-20T19:40:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:21 smithi118 ceph-mon[49197]: Reconfiguring osd.2 (monmap changed)... 2024-09-20T19:40:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:21 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.2 on smithi116 2024-09-20T19:40:22.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 138161 10.81 MB/sec execute 192 sec latency 53.766 ms 2024-09-20T19:40:22.325 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:22 smithi116 ceph-mon[63578]: pgmap v22: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-20T19:40:22.325 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:22.325 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:22.325 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T19:40:22.325 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:22.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:22 smithi118 ceph-mon[49197]: pgmap v22: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-20T19:40:22.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:22.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:22.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T19:40:22.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:23.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 138657 10.85 MB/sec execute 193 sec latency 64.025 ms 2024-09-20T19:40:23.331 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:23 smithi116 ceph-mon[63578]: Reconfiguring osd.3 (monmap changed)... 2024-09-20T19:40:23.331 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:23 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.3 on smithi116 2024-09-20T19:40:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:23 smithi118 ceph-mon[49197]: Reconfiguring osd.3 (monmap changed)... 2024-09-20T19:40:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:23 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.3 on smithi116 2024-09-20T19:40:24.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 138992 10.85 MB/sec execute 194 sec latency 95.023 ms 2024-09-20T19:40:24.351 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:24 smithi116 ceph-mon[63578]: pgmap v23: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-20T19:40:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:24 smithi118 ceph-mon[49197]: pgmap v23: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-20T19:40:25.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 139286 10.80 MB/sec execute 195 sec latency 56.797 ms 2024-09-20T19:40:26.153 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:25 smithi116 ceph-mon[63578]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.1 MiB/s wr, 673 op/s 2024-09-20T19:40:26.153 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:26.153 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:26.153 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:25 smithi116 ceph-mon[63578]: Reconfiguring mds.foofs.smithi116.xvfgaf (monmap changed)... 2024-09-20T19:40:26.153 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:40:26.154 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:26.154 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:25 smithi116 ceph-mon[63578]: Reconfiguring daemon mds.foofs.smithi116.xvfgaf on smithi116 2024-09-20T19:40:26.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:25 smithi118 ceph-mon[49197]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.1 MiB/s wr, 673 op/s 2024-09-20T19:40:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:25 smithi118 ceph-mon[49197]: Reconfiguring mds.foofs.smithi116.xvfgaf (monmap changed)... 2024-09-20T19:40:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:40:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:25 smithi118 ceph-mon[49197]: Reconfiguring daemon mds.foofs.smithi116.xvfgaf on smithi116 2024-09-20T19:40:26.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 139492 10.77 MB/sec execute 196 sec latency 64.312 ms 2024-09-20T19:40:27.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 139940 10.78 MB/sec execute 197 sec latency 49.042 ms 2024-09-20T19:40:28.188 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-20T19:40:28.188 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:28.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:28.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-20T19:40:28.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:28.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:28.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-20T19:40:28.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:28.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 140292 10.77 MB/sec execute 198 sec latency 90.403 ms 2024-09-20T19:40:28.391 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-20T19:40:28.392 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:28.392 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:28.392 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: Reconfiguring crash.smithi118 (monmap changed)... 2024-09-20T19:40:28.392 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:28.392 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:28.392 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: Reconfiguring daemon crash.smithi118 on smithi118 2024-09-20T19:40:28.392 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:29.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 140665 10.76 MB/sec execute 199 sec latency 51.159 ms 2024-09-20T19:40:30.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: pgmap v26: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-09-20T19:40:30.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:30.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:30.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: Reconfiguring mgr.smithi118.rmiazv (monmap changed)... 2024-09-20T19:40:30.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T19:40:30.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T19:40:30.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:30.266 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:30 smithi116 ceph-mon[63578]: Reconfiguring daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:40:30.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 140935 10.73 MB/sec execute 200 sec latency 59.310 ms 2024-09-20T19:40:30.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: pgmap v26: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-09-20T19:40:30.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:30.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:30.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: Reconfiguring mgr.smithi118.rmiazv (monmap changed)... 2024-09-20T19:40:30.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi118.rmiazv", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T19:40:30.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T19:40:30.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:30.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:30 smithi118 ceph-mon[49197]: Reconfiguring daemon mgr.smithi118.rmiazv on smithi118 2024-09-20T19:40:31.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 141206 10.71 MB/sec execute 201 sec latency 79.353 ms 2024-09-20T19:40:31.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-09-20T19:40:31.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:31.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:31.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-20T19:40:31.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T19:40:31.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T19:40:31.844 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:31.844 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:31 smithi116 ceph-mon[63578]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-20T19:40:31.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-09-20T19:40:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: Reconfiguring mon.smithi118 (monmap changed)... 2024-09-20T19:40:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T19:40:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T19:40:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:31 smithi118 ceph-mon[49197]: Reconfiguring daemon mon.smithi118 on smithi118 2024-09-20T19:40:32.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 141598 10.73 MB/sec execute 202 sec latency 57.773 ms 2024-09-20T19:40:33.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 142093 10.78 MB/sec execute 203 sec latency 50.172 ms 2024-09-20T19:40:33.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:33 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:33.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:33 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:33.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:33 smithi116 ceph-mon[63578]: Reconfiguring osd.4 (monmap changed)... 2024-09-20T19:40:33.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:33 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T19:40:33.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:33 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:33.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:33 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.4 on smithi118 2024-09-20T19:40:33.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:33 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:33.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:33 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:33.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:33 smithi118 ceph-mon[49197]: Reconfiguring osd.4 (monmap changed)... 2024-09-20T19:40:33.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:33 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T19:40:33.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:33 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:33.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:33 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.4 on smithi118 2024-09-20T19:40:34.275 INFO:teuthology.orchestra.run.smithi116.stdout: 5 142352 10.76 MB/sec execute 204 sec latency 51.732 ms 2024-09-20T19:40:34.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:34 smithi116 ceph-mon[63578]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-09-20T19:40:34.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:34.399 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:34.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T19:40:34.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:34.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:34 smithi118 ceph-mon[49197]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 470 op/s 2024-09-20T19:40:34.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:34.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:34.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T19:40:34.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:35.124 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:40:35.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:35 smithi116 ceph-mon[63578]: Reconfiguring osd.5 (monmap changed)... 2024-09-20T19:40:35.246 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:35 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.5 on smithi118 2024-09-20T19:40:35.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 142765 10.74 MB/sec execute 205 sec latency 38.623 ms 2024-09-20T19:40:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:35 smithi118 ceph-mon[49197]: Reconfiguring osd.5 (monmap changed)... 2024-09-20T19:40:35.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:35 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.5 on smithi118 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (3m) 45s ago 11m 15.6M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (11m) 45s ago 11m 6433k - 17.2.0 e1d6a67b021e 5d04e1b9f54d 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (11m) 32s ago 11m 6400k - 17.2.0 e1d6a67b021e 1b745626ca1f 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (111s) 45s ago 11m 43.5M - dad864ee21e9 8a163a77164a 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (5m) 45s ago 5m 36.9M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (5m) 32s ago 5m 14.0M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (2m) 45s ago 13m 468M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (83s) 32s ago 11m 446M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:40:35.994 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (64s) 45s ago 13m 32.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (37s) 32s ago 11m 25.1M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (5m) 45s ago 5m 280M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (5m) 32s ago 5m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (2m) 45s ago 11m 9726k - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (2m) 32s ago 11m 11.3M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (10m) 45s ago 10m 461M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (9m) 45s ago 9m 398M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (9m) 45s ago 9m 271M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (9m) 45s ago 9m 425M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (8m) 32s ago 8m 480M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (8m) 32s ago 8m 512M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (7m) 32s ago 7m 430M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (7m) 32s ago 7m 576M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:40:35.995 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (89s) 45s ago 10m 46.2M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:40:36.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 143044 10.71 MB/sec execute 206 sec latency 41.268 ms 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 679 op/s 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: Reconfiguring osd.6 (monmap changed)... 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:36.450 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:36 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.6 on smithi118 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:40:36.553 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:40:36.554 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:40:36.554 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:40:36.554 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T19:40:36.554 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 4 2024-09-20T19:40:36.554 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:40:36.554 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:40:36.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: pgmap v29: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 679 op/s 2024-09-20T19:40:36.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:36.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:36.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:36.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: Reconfiguring osd.6 (monmap changed)... 2024-09-20T19:40:36.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T19:40:36.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:36.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:36 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.6 on smithi118 2024-09-20T19:40:37.144 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr" 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "4/23 daemons upgraded", 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:40:37.145 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:40:37.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 143409 10.71 MB/sec execute 207 sec latency 56.952 ms 2024-09-20T19:40:37.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:37 smithi116 ceph-mon[63578]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:37.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:37 smithi116 ceph-mon[63578]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:37.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:37 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/470923088' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:37.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:37 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:37.458 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:37 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:37.458 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:37 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T19:40:37.458 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:37 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:37.592 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:37 smithi118 ceph-mon[49197]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:37.592 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:37 smithi118 ceph-mon[49197]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:37.592 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:37 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/470923088' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:37.592 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:37 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:37.593 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:37 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:37.593 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:37 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T19:40:37.593 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:37 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:37.808 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 mgr modules have recently crashed 2024-09-20T19:40:37.808 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:40:37.808 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:40:37.808 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 1 mgr modules have recently crashed 2024-09-20T19:40:37.808 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:40:38.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 143727 10.69 MB/sec execute 208 sec latency 64.620 ms 2024-09-20T19:40:38.486 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:38 smithi116 ceph-mon[63578]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-20T19:40:38.486 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:38 smithi116 ceph-mon[63578]: Reconfiguring osd.7 (monmap changed)... 2024-09-20T19:40:38.486 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:38 smithi116 ceph-mon[63578]: Reconfiguring daemon osd.7 on smithi118 2024-09-20T19:40:38.487 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:38 smithi116 ceph-mon[63578]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:38.487 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:38 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/512010886' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:40:38.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:38 smithi118 ceph-mon[49197]: pgmap v30: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-20T19:40:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:38 smithi118 ceph-mon[49197]: Reconfiguring osd.7 (monmap changed)... 2024-09-20T19:40:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:38 smithi118 ceph-mon[49197]: Reconfiguring daemon osd.7 on smithi118 2024-09-20T19:40:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:38 smithi118 ceph-mon[49197]: from='client.44117 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:38 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/512010886' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:40:39.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 144091 10.69 MB/sec execute 209 sec latency 45.473 ms 2024-09-20T19:40:39.505 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:39.505 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:39.506 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:39 smithi116 ceph-mon[63578]: Reconfiguring mds.foofs.smithi118.zumcyi (monmap changed)... 2024-09-20T19:40:39.506 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:40:39.506 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:39 smithi116 ceph-mon[63578]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 626 op/s 2024-09-20T19:40:39.506 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:39.506 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:39 smithi116 ceph-mon[63578]: Reconfiguring daemon mds.foofs.smithi118.zumcyi on smithi118 2024-09-20T19:40:39.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:39 smithi118 ceph-mon[49197]: Reconfiguring mds.foofs.smithi118.zumcyi (monmap changed)... 2024-09-20T19:40:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:40:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:39 smithi118 ceph-mon[49197]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 626 op/s 2024-09-20T19:40:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:39 smithi118 ceph-mon[49197]: Reconfiguring daemon mds.foofs.smithi118.zumcyi on smithi118 2024-09-20T19:40:40.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 144376 10.67 MB/sec execute 210 sec latency 59.878 ms 2024-09-20T19:40:41.129 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all mon 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi116"} : dispatch 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi116"}]': finished 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"} : dispatch 2024-09-20T19:40:41.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"}]': finished 2024-09-20T19:40:41.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 144735 10.66 MB/sec execute 211 sec latency 46.064 ms 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all mon 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi116"} : dispatch 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi116"}]': finished 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"} : dispatch 2024-09-20T19:40:41.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi118"}]': finished 2024-09-20T19:40:42.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 145247 10.69 MB/sec execute 212 sec latency 36.793 ms 2024-09-20T19:40:42.582 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:42 smithi116 ceph-mon[63578]: pgmap v32: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-20T19:40:42.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:42 smithi118 ceph-mon[49197]: pgmap v32: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-20T19:40:43.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 145685 10.71 MB/sec execute 213 sec latency 30.902 ms 2024-09-20T19:40:43.588 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:43.588 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:43.588 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:43.588 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi116", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:44.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 146070 10.73 MB/sec execute 214 sec latency 41.277 ms 2024-09-20T19:40:44.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:44 smithi116 ceph-mon[63578]: pgmap v33: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-20T19:40:44.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:44 smithi116 ceph-mon[63578]: Upgrade: Updating crash.smithi116 (1/2) 2024-09-20T19:40:44.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:44 smithi116 ceph-mon[63578]: Deploying daemon crash.smithi116 on smithi116 2024-09-20T19:40:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:44 smithi118 ceph-mon[49197]: pgmap v33: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-20T19:40:44.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:44 smithi118 ceph-mon[49197]: Upgrade: Updating crash.smithi116 (1/2) 2024-09-20T19:40:44.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:44 smithi118 ceph-mon[49197]: Deploying daemon crash.smithi116 on smithi116 2024-09-20T19:40:45.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 146420 10.69 MB/sec execute 215 sec latency 56.157 ms 2024-09-20T19:40:46.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 146756 10.68 MB/sec execute 216 sec latency 49.231 ms 2024-09-20T19:40:46.661 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:46 smithi116 ceph-mon[63578]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.8 MiB/s wr, 643 op/s 2024-09-20T19:40:46.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:46 smithi118 ceph-mon[49197]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.8 MiB/s wr, 643 op/s 2024-09-20T19:40:47.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 147080 10.67 MB/sec execute 217 sec latency 51.732 ms 2024-09-20T19:40:48.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 147396 10.65 MB/sec execute 218 sec latency 72.442 ms 2024-09-20T19:40:48.458 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:48 smithi116 ceph-mon[63578]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 435 op/s 2024-09-20T19:40:48.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:48 smithi118 ceph-mon[49197]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 435 op/s 2024-09-20T19:40:49.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 147805 10.66 MB/sec execute 219 sec latency 48.824 ms 2024-09-20T19:40:49.975 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:49 smithi116 ceph-mon[63578]: pgmap v36: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 654 op/s 2024-09-20T19:40:49.975 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:49 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:49.975 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:49 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:49.975 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:49 smithi116 ceph-mon[63578]: from='client.44123 -' entity='client.crash.smithi116' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:49.975 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:49 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:49.976 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:49 smithi116 ceph-mon[63578]: pgmap v37: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s 2024-09-20T19:40:49.976 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:49 smithi116 ceph-mon[63578]: Health check update: 2 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH) 2024-09-20T19:40:50.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:49 smithi118 ceph-mon[49197]: pgmap v36: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 654 op/s 2024-09-20T19:40:50.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:49 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:50.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:49 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:50.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:49 smithi118 ceph-mon[49197]: from='client.44123 -' entity='client.crash.smithi116' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:50.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:49 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:50.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:49 smithi118 ceph-mon[49197]: pgmap v37: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s 2024-09-20T19:40:50.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:49 smithi118 ceph-mon[49197]: Health check update: 2 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH) 2024-09-20T19:40:50.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 148223 10.65 MB/sec execute 220 sec latency 41.520 ms 2024-09-20T19:40:51.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 148550 10.66 MB/sec execute 221 sec latency 54.752 ms 2024-09-20T19:40:51.526 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:51 smithi116 ceph-mon[63578]: Upgrade: Updating crash.smithi118 (2/2) 2024-09-20T19:40:51.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:51.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:51.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:51.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:51 smithi116 ceph-mon[63578]: Deploying daemon crash.smithi118 on smithi118 2024-09-20T19:40:51.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:51 smithi118 ceph-mon[49197]: Upgrade: Updating crash.smithi118 (2/2) 2024-09-20T19:40:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:51 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:51 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi118", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T19:40:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:51 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:40:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:51 smithi118 ceph-mon[49197]: Deploying daemon crash.smithi118 on smithi118 2024-09-20T19:40:52.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 148956 10.67 MB/sec execute 222 sec latency 55.219 ms 2024-09-20T19:40:52.534 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:52 smithi116 ceph-mon[63578]: pgmap v38: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s 2024-09-20T19:40:52.629 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:52 smithi118 ceph-mon[49197]: pgmap v38: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 499 op/s 2024-09-20T19:40:53.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 149514 10.72 MB/sec execute 223 sec latency 40.219 ms 2024-09-20T19:40:54.276 INFO:teuthology.orchestra.run.smithi116.stdout: 5 150172 10.71 MB/sec execute 224 sec latency 34.118 ms 2024-09-20T19:40:54.448 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:54 smithi118 ceph-mon[49197]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 764 op/s 2024-09-20T19:40:54.544 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:54 smithi116 ceph-mon[63578]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 764 op/s 2024-09-20T19:40:55.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 150710 10.72 MB/sec execute 225 sec latency 19.942 ms 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='client.44127 -' entity='client.crash.smithi118' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='client.44131 -' entity='client.crash.smithi118' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: pgmap v41: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.4 MiB/s wr, 620 op/s 2024-09-20T19:40:55.808 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:55 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:56.090 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='client.44127 -' entity='client.crash.smithi118' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='client.44131 -' entity='client.crash.smithi118' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: pgmap v41: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.4 MiB/s wr, 620 op/s 2024-09-20T19:40:56.091 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:55 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:40:56.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 151223 10.73 MB/sec execute 226 sec latency 53.120 ms 2024-09-20T19:40:56.812 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:56 smithi116 ceph-mon[63578]: from='client.44135 -' entity='client.crash.smithi118' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:56.812 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:56 smithi116 ceph-mon[63578]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 397 op/s 2024-09-20T19:40:56.812 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:56 smithi116 ceph-mon[63578]: Health check update: 5 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH) 2024-09-20T19:40:57.037 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:56 smithi118 ceph-mon[49197]: from='client.44135 -' entity='client.crash.smithi118' cmd=[{"prefix": "crash post", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:40:57.037 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:56 smithi118 ceph-mon[49197]: pgmap v42: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 397 op/s 2024-09-20T19:40:57.037 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:56 smithi118 ceph-mon[49197]: Health check update: 5 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH) 2024-09-20T19:40:57.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 151745 10.74 MB/sec execute 227 sec latency 57.722 ms 2024-09-20T19:40:57.819 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:58.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:40:58.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 152211 10.77 MB/sec execute 228 sec latency 34.565 ms 2024-09-20T19:40:58.830 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:58 smithi116 ceph-mon[63578]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.8 MiB/s wr, 335 op/s 2024-09-20T19:40:59.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:58 smithi118 ceph-mon[49197]: pgmap v43: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.8 MiB/s wr, 335 op/s 2024-09-20T19:40:59.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 152608 10.78 MB/sec execute 229 sec latency 41.553 ms 2024-09-20T19:41:00.101 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:00.101 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:00.101 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:00.101 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:40:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:00.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 152867 10.77 MB/sec execute 230 sec latency 104.943 ms 2024-09-20T19:41:00.312 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:00.312 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:00.312 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:00.312 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:40:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:01.144 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:00 smithi116 ceph-mon[63578]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 800 op/s 2024-09-20T19:41:01.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:00 smithi118 ceph-mon[49197]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 800 op/s 2024-09-20T19:41:01.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 153152 10.76 MB/sec execute 231 sec latency 66.721 ms 2024-09-20T19:41:02.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 153560 10.74 MB/sec execute 232 sec latency 54.927 ms 2024-09-20T19:41:03.160 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:02 smithi116 ceph-mon[63578]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 465 op/s 2024-09-20T19:41:03.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:02 smithi118 ceph-mon[49197]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.5 MiB/s wr, 465 op/s 2024-09-20T19:41:03.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 153939 10.72 MB/sec execute 233 sec latency 44.537 ms 2024-09-20T19:41:04.179 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:04.179 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:04.179 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:04.179 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:04.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 154291 10.72 MB/sec execute 234 sec latency 44.505 ms 2024-09-20T19:41:04.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:04.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:05.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:05 smithi116 ceph-mon[63578]: pgmap v46: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 762 op/s 2024-09-20T19:41:05.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:05 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:05.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:05 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:05.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 154598 10.71 MB/sec execute 235 sec latency 74.740 ms 2024-09-20T19:41:05.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:05 smithi118 ceph-mon[49197]: pgmap v46: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 762 op/s 2024-09-20T19:41:05.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:05 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:05.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:05 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:06.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 155251 10.73 MB/sec execute 236 sec latency 26.538 ms 2024-09-20T19:41:07.222 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:07 smithi116 ceph-mon[63578]: pgmap v47: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.3 MiB/s wr, 637 op/s 2024-09-20T19:41:07.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 155892 10.78 MB/sec execute 237 sec latency 19.963 ms 2024-09-20T19:41:07.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:07 smithi118 ceph-mon[49197]: pgmap v47: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.3 MiB/s wr, 637 op/s 2024-09-20T19:41:08.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 156494 10.81 MB/sec execute 238 sec latency 30.916 ms 2024-09-20T19:41:08.387 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:41:09.273 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:09 smithi116 ceph-mon[63578]: pgmap v48: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-20T19:41:09.273 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:09 smithi116 ceph-mon[63578]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:09.277 INFO:teuthology.orchestra.run.smithi116.stdout: 5 156990 10.82 MB/sec execute 239 sec latency 44.778 ms 2024-09-20T19:41:09.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:09 smithi118 ceph-mon[49197]: pgmap v48: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-20T19:41:09.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:09 smithi118 ceph-mon[49197]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:09.475 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (3m) 6s ago 12m 15.6M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (21s) 6s ago 12m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (15s) 10s ago 11m 37.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (2m) 6s ago 12m 44.4M - dad864ee21e9 8a163a77164a 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (6m) 6s ago 6m 36.8M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (6m) 10s ago 6m 14.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (2m) 6s ago 13m 490M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (116s) 10s ago 11m 448M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (98s) 6s ago 13m 43.6M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (71s) 10s ago 11m 38.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (6m) 6s ago 6m 276M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:41:09.476 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (6m) 10s ago 6m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (3m) 6s ago 12m 9919k - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (2m) 10s ago 11m 10.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (10m) 6s ago 10m 536M 1936M 17.2.0 e1d6a67b021e d75f3decccb4 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (10m) 6s ago 10m 463M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (10m) 6s ago 10m 320M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (9m) 6s ago 9m 492M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (9m) 10s ago 9m 517M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (8m) 10s ago 8m 318M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (8m) 10s ago 8m 512M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (8m) 10s ago 8m 383M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:41:09.477 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (2m) 6s ago 11m 48.6M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:41:10.160 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:41:10.160 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:41:10.160 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:41:10.160 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:10.161 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:41:10.161 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:41:10.161 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:10.161 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:41:10.161 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T19:41:10.161 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:10.161 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:41:10.162 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:41:10.162 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:10.162 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:41:10.162 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T19:41:10.162 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 4 2024-09-20T19:41:10.162 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:41:10.162 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:41:10.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 157313 10.80 MB/sec execute 240 sec latency 51.598 ms 2024-09-20T19:41:10.295 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:10 smithi116 ceph-mon[63578]: from='client.44143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:10.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:10 smithi118 ceph-mon[49197]: from='client.44143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:10.694 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:41:10.694 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:41:10.695 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:41:10.695 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:41:10.695 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:41:10.695 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:41:10.695 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:41:10.695 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:41:10.696 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:41:10.696 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "6/23 daemons upgraded", 2024-09-20T19:41:10.696 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading crash daemons", 2024-09-20T19:41:10.696 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:41:10.696 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:41:11.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 157687 10.79 MB/sec execute 241 sec latency 51.804 ms 2024-09-20T19:41:11.297 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:41:11.298 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: pgmap v49: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 839 op/s 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2543082963' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:41:11.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi116"} : dispatch 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi116"}]': finished 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"} : dispatch 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"}]': finished 2024-09-20T19:41:11.311 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: pgmap v49: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 839 op/s 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2543082963' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi116"} : dispatch 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi116"}]': finished 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"} : dispatch 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi118"}]': finished 2024-09-20T19:41:11.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-20T19:41:12.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 158084 10.78 MB/sec execute 242 sec latency 39.466 ms 2024-09-20T19:41:12.327 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:12 smithi116 ceph-mon[63578]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:12.327 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:12 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all crash 2024-09-20T19:41:12.327 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:12 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T19:41:12.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:12 smithi116 ceph-mon[63578]: Upgrade: osd.0 is safe to restart 2024-09-20T19:41:12.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:12 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/3360512531' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:41:12.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:12 smithi118 ceph-mon[49197]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:12.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:12 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all crash 2024-09-20T19:41:12.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:12 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T19:41:12.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:12 smithi118 ceph-mon[49197]: Upgrade: osd.0 is safe to restart 2024-09-20T19:41:12.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:12 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/3360512531' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:41:13.098 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:13 smithi116 ceph-mon[63578]: pgmap v50: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 511 op/s 2024-09-20T19:41:13.099 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:13.099 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:13.099 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T19:41:13.099 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:41:13.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 158501 10.80 MB/sec execute 243 sec latency 57.457 ms 2024-09-20T19:41:13.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:13 smithi118 ceph-mon[49197]: pgmap v50: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 511 op/s 2024-09-20T19:41:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T19:41:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:41:14.143 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:14 smithi116 ceph-mon[63578]: Upgrade: Updating osd.0 2024-09-20T19:41:14.143 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:14 smithi116 ceph-mon[63578]: Deploying daemon osd.0 on smithi116 2024-09-20T19:41:14.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 158932 10.80 MB/sec execute 244 sec latency 64.129 ms 2024-09-20T19:41:14.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:14 smithi118 ceph-mon[49197]: Upgrade: Updating osd.0 2024-09-20T19:41:14.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:14 smithi118 ceph-mon[49197]: Deploying daemon osd.0 on smithi116 2024-09-20T19:41:15.153 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:15 smithi116 ceph-mon[63578]: pgmap v51: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 768 op/s 2024-09-20T19:41:15.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 159290 10.79 MB/sec execute 245 sec latency 37.009 ms 2024-09-20T19:41:15.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:15 smithi118 ceph-mon[49197]: pgmap v51: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 768 op/s 2024-09-20T19:41:16.186 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:16 smithi116 ceph-mon[63578]: mgrmap e43: smithi116.poxcrq(active, since 93s), standbys: smithi118.rmiazv 2024-09-20T19:41:16.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 159626 10.80 MB/sec execute 246 sec latency 39.402 ms 2024-09-20T19:41:16.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:16 smithi118 ceph-mon[49197]: mgrmap e43: smithi116.poxcrq(active, since 93s), standbys: smithi118.rmiazv 2024-09-20T19:41:17.201 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:17 smithi116 ceph-mon[63578]: pgmap v52: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-20T19:41:17.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 159931 10.80 MB/sec execute 247 sec latency 47.395 ms 2024-09-20T19:41:17.459 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:17 smithi116 systemd[1]: Stopping Ceph osd.0 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:41:17.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:17 smithi118 ceph-mon[49197]: pgmap v52: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-20T19:41:17.980 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:17 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[34297]: 2024-09-20T19:41:17.722+0000 7fddfde6f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:41:17.980 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:17 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[34297]: 2024-09-20T19:41:17.722+0000 7fddfde6f700 -1 osd.0 73 *** Got signal Terminated *** 2024-09-20T19:41:17.980 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:17 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[34297]: 2024-09-20T19:41:17.722+0000 7fddfde6f700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:41:18.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:18 smithi116 ceph-mon[63578]: osd.0 marked itself down and dead 2024-09-20T19:41:18.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 160206 10.79 MB/sec execute 248 sec latency 145.268 ms 2024-09-20T19:41:18.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:18 smithi118 ceph-mon[49197]: osd.0 marked itself down and dead 2024-09-20T19:41:19.052 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:18 smithi116 podman[67895]: 2024-09-20 19:41:18.79734183 +0000 UTC m=+1.275306995 container died d75f3decccb4a79400d419018e454fe6dca91a6a6ae8eb03b3a369bc096e8f0c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, RELEASE=HEAD, ceph=True, vcs-type=git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.openshift.tags=base centos centos-stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-20T19:41:19.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 160314 10.76 MB/sec execute 249 sec latency 1019.797 ms 2024-09-20T19:41:19.315 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:19 smithi116 ceph-mon[63578]: pgmap v53: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-20T19:41:19.315 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:19 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:41:19.315 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:19 smithi116 ceph-mon[63578]: osdmap e74: 8 total, 7 up, 8 in 2024-09-20T19:41:19.316 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:19 smithi116 podman[67895]: 2024-09-20 19:41:19.200569282 +0000 UTC m=+1.678534433 container cleanup d75f3decccb4a79400d419018e454fe6dca91a6a6ae8eb03b3a369bc096e8f0c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, distribution-scope=public, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-20T19:41:19.316 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:19 smithi116 bash[67895]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0 2024-09-20T19:41:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:19 smithi118 ceph-mon[49197]: pgmap v53: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-20T19:41:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:19 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:41:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:19 smithi118 ceph-mon[49197]: osdmap e74: 8 total, 7 up, 8 in 2024-09-20T19:41:19.573 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:19 smithi116 podman[67911]: 2024-09-20 19:41:19.555792807 +0000 UTC m=+0.755455299 container remove d75f3decccb4a79400d419018e454fe6dca91a6a6ae8eb03b3a369bc096e8f0c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-type=git, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, maintainer=Guillaume Abrioux , architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, version=8) 2024-09-20T19:41:20.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 160624 10.73 MB/sec execute 250 sec latency 60.661 ms 2024-09-20T19:41:20.339 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:20 smithi116 ceph-mon[63578]: osdmap e75: 8 total, 7 up, 8 in 2024-09-20T19:41:20.340 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:20 smithi116 podman[67994]: 2024-09-20 19:41:20.015568064 +0000 UTC m=+0.022530288 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:41:20.340 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:20 smithi116 podman[67994]: 2024-09-20 19:41:20.163365657 +0000 UTC m=+0.170327865 container create 098308dd4be197819cbd1c8f589db381c6e7502a8bbc016feda3ce17d7a697c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0) 2024-09-20T19:41:20.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:20 smithi118 ceph-mon[49197]: osdmap e75: 8 total, 7 up, 8 in 2024-09-20T19:41:20.591 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:20 smithi116 podman[67994]: 2024-09-20 19:41:20.500601651 +0000 UTC m=+0.507563869 container init 098308dd4be197819cbd1c8f589db381c6e7502a8bbc016feda3ce17d7a697c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:41:20.591 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:20 smithi116 podman[67994]: 2024-09-20 19:41:20.505597154 +0000 UTC m=+0.512559367 container start 098308dd4be197819cbd1c8f589db381c6e7502a8bbc016feda3ce17d7a697c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:41:21.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 160961 10.72 MB/sec execute 251 sec latency 81.621 ms 2024-09-20T19:41:21.351 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:21 smithi116 ceph-mon[63578]: pgmap v56: 97 pgs: 27 peering, 70 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 696 op/s 2024-09-20T19:41:21.351 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:21 smithi116 ceph-mon[63578]: Health check failed: Reduced data availability: 3 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-20T19:41:21.352 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:21 smithi116 podman[67994]: 2024-09-20 19:41:21.123972228 +0000 UTC m=+1.130934444 container attach 098308dd4be197819cbd1c8f589db381c6e7502a8bbc016feda3ce17d7a697c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:41:21.352 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:21 smithi116 podman[67994]: 2024-09-20 19:41:21.124259766 +0000 UTC m=+1.131221984 container died 098308dd4be197819cbd1c8f589db381c6e7502a8bbc016feda3ce17d7a697c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:41:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:21 smithi118 ceph-mon[49197]: pgmap v56: 97 pgs: 27 peering, 70 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 696 op/s 2024-09-20T19:41:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:21 smithi118 ceph-mon[49197]: Health check failed: Reduced data availability: 3 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-20T19:41:22.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 161413 10.71 MB/sec execute 252 sec latency 49.370 ms 2024-09-20T19:41:22.620 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:22 smithi116 podman[68017]: 2024-09-20 19:41:22.402948971 +0000 UTC m=+1.567308780 container remove 098308dd4be197819cbd1c8f589db381c6e7502a8bbc016feda3ce17d7a697c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-20T19:41:22.620 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:22 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-20T19:41:22.620 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:22 smithi116 systemd[1]: Stopped Ceph osd.0 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:41:22.620 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:22 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.0.service: Consumed 2min 34.675s CPU time. 2024-09-20T19:41:22.889 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:22 smithi116 systemd[1]: Starting Ceph osd.0 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:41:23.139 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 podman[68115]: 2024-09-20 19:41:22.98901357 +0000 UTC m=+0.025440060 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:41:23.139 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 podman[68115]: 2024-09-20 19:41:23.121671047 +0000 UTC m=+0.158097521 container create c9e9f4b3ae0d829b6d42638fb06c06bee671358e7322fc84d0449664751fd955 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-20T19:41:23.278 INFO:teuthology.orchestra.run.smithi116.stdout: 5 161770 10.71 MB/sec execute 253 sec latency 42.684 ms 2024-09-20T19:41:23.392 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:23 smithi116 ceph-mon[63578]: pgmap v57: 97 pgs: 27 peering, 70 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.6 MiB/s wr, 311 op/s 2024-09-20T19:41:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:23 smithi118 ceph-mon[49197]: pgmap v57: 97 pgs: 27 peering, 70 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.6 MiB/s wr, 311 op/s 2024-09-20T19:41:23.654 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 podman[68115]: 2024-09-20 19:41:23.492163413 +0000 UTC m=+0.528589904 container init c9e9f4b3ae0d829b6d42638fb06c06bee671358e7322fc84d0449664751fd955 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0) 2024-09-20T19:41:23.654 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 podman[68115]: 2024-09-20 19:41:23.499566487 +0000 UTC m=+0.535992958 container start c9e9f4b3ae0d829b6d42638fb06c06bee671358e7322fc84d0449664751fd955 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-20T19:41:23.915 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:41:23.915 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 bash[68115]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:41:23.915 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 podman[68115]: 2024-09-20 19:41:23.665105101 +0000 UTC m=+0.701531580 container attach c9e9f4b3ae0d829b6d42638fb06c06bee671358e7322fc84d0449664751fd955 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:41:23.915 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:41:23.915 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:23 smithi116 bash[68115]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:41:24.288 INFO:teuthology.orchestra.run.smithi116.stdout: 5 162255 10.73 MB/sec execute 254 sec latency 43.072 ms 2024-09-20T19:41:24.418 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:24 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 229/1731 objects degraded (13.229%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:24.419 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:24 smithi116 ceph-mon[63578]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 9 pgs peering) 2024-09-20T19:41:24.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:24 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 229/1731 objects degraded (13.229%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:24 smithi118 ceph-mon[49197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 9 pgs peering) 2024-09-20T19:41:25.176 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T19:41:25.176 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 bash[68115]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T19:41:25.176 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T19:41:25.177 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 bash[68115]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T19:41:25.288 INFO:teuthology.orchestra.run.smithi116.stdout: 5 162635 10.72 MB/sec execute 255 sec latency 42.591 ms 2024-09-20T19:41:25.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-mon[63578]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 602 op/s; 229/1731 objects degraded (13.229%) 2024-09-20T19:41:25.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:25 smithi118 ceph-mon[49197]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 602 op/s; 229/1731 objects degraded (13.229%) 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 bash[68115]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 bash[68115]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 bash[68115]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 bash[68115]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate[68129]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-20T19:41:25.688 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 bash[68115]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-20T19:41:25.689 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:25 smithi116 podman[68115]: 2024-09-20 19:41:25.557516055 +0000 UTC m=+2.593942538 container died c9e9f4b3ae0d829b6d42638fb06c06bee671358e7322fc84d0449664751fd955 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:41:26.288 INFO:teuthology.orchestra.run.smithi116.stdout: 5 162936 10.72 MB/sec execute 256 sec latency 58.828 ms 2024-09-20T19:41:26.701 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:26 smithi116 podman[68268]: 2024-09-20 19:41:26.594891521 +0000 UTC m=+1.027089973 container remove c9e9f4b3ae0d829b6d42638fb06c06bee671358e7322fc84d0449664751fd955 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-20T19:41:26.955 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:26 smithi116 podman[68328]: 2024-09-20 19:41:26.736929321 +0000 UTC m=+0.019407748 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:41:26.955 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:26 smithi116 podman[68328]: 2024-09-20 19:41:26.860930145 +0000 UTC m=+0.143408553 container create cbf89945fd9e87414fd895d1d513bf801f66ac5ea9f5946e49b8c4f31fbb80e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-20T19:41:27.211 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:27 smithi116 podman[68328]: 2024-09-20 19:41:27.197288224 +0000 UTC m=+0.479766632 container init cbf89945fd9e87414fd895d1d513bf801f66ac5ea9f5946e49b8c4f31fbb80e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-20T19:41:27.212 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:27 smithi116 podman[68328]: 2024-09-20 19:41:27.20248028 +0000 UTC m=+0.484958695 container start cbf89945fd9e87414fd895d1d513bf801f66ac5ea9f5946e49b8c4f31fbb80e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux ) 2024-09-20T19:41:27.288 INFO:teuthology.orchestra.run.smithi116.stdout: 5 163314 10.73 MB/sec execute 257 sec latency 44.948 ms 2024-09-20T19:41:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:27 smithi118 ceph-mon[49197]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 602 op/s; 229/1731 objects degraded (13.229%) 2024-09-20T19:41:27.467 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:27 smithi116 ceph-mon[63578]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 602 op/s; 229/1731 objects degraded (13.229%) 2024-09-20T19:41:27.467 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:27 smithi116 ceph-osd[68346]: -- 172.21.15.116:0/378476631 <== mon.0 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b968b8c1e0 con 0x55b967d27c00 2024-09-20T19:41:27.467 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:27 smithi116 bash[68328]: cbf89945fd9e87414fd895d1d513bf801f66ac5ea9f5946e49b8c4f31fbb80e0 2024-09-20T19:41:27.468 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:27 smithi116 systemd[1]: Started Ceph osd.0 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:41:28.289 INFO:teuthology.orchestra.run.smithi116.stdout: 5 163674 10.72 MB/sec execute 258 sec latency 48.179 ms 2024-09-20T19:41:28.471 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:28.471 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:28.471 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:41:28.471 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:28.471 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:28.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:41:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:29.289 INFO:teuthology.orchestra.run.smithi116.stdout: 5 163957 10.71 MB/sec execute 259 sec latency 59.606 ms 2024-09-20T19:41:29.509 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:29 smithi116 ceph-mon[63578]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 504 op/s; 229/1731 objects degraded (13.229%) 2024-09-20T19:41:29.510 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:29 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[68342]: 2024-09-20T19:41:29.311+0000 7f1bd157d640 -1 Falling back to public interface 2024-09-20T19:41:29.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:29 smithi118 ceph-mon[49197]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 504 op/s; 229/1731 objects degraded (13.229%) 2024-09-20T19:41:30.289 INFO:teuthology.orchestra.run.smithi116.stdout: 5 164205 10.69 MB/sec execute 260 sec latency 71.176 ms 2024-09-20T19:41:30.520 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:30 smithi116 ceph-mon[63578]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 218/1740 objects degraded (12.529%) 2024-09-20T19:41:30.520 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:30 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 218/1740 objects degraded (12.529%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:30.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:30 smithi118 ceph-mon[49197]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 218/1740 objects degraded (12.529%) 2024-09-20T19:41:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:30 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 218/1740 objects degraded (12.529%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:31.289 INFO:teuthology.orchestra.run.smithi116.stdout: 5 164436 10.67 MB/sec execute 261 sec latency 113.220 ms 2024-09-20T19:41:32.297 INFO:teuthology.orchestra.run.smithi116.stdout: 5 164803 10.66 MB/sec execute 262 sec latency 48.870 ms 2024-09-20T19:41:32.826 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:32 smithi116 ceph-mon[63578]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.3 MiB/s wr, 417 op/s; 218/1740 objects degraded (12.529%) 2024-09-20T19:41:32.826 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:32 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:33.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:32 smithi118 ceph-mon[49197]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.3 MiB/s wr, 417 op/s; 218/1740 objects degraded (12.529%) 2024-09-20T19:41:33.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:32 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:33.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 165175 10.64 MB/sec execute 263 sec latency 35.207 ms 2024-09-20T19:41:34.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 165446 10.64 MB/sec execute 264 sec latency 51.734 ms 2024-09-20T19:41:34.892 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:34 smithi116 ceph-mon[63578]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 606 op/s; 215/1698 objects degraded (12.662%) 2024-09-20T19:41:35.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:34 smithi118 ceph-mon[49197]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 606 op/s; 215/1698 objects degraded (12.662%) 2024-09-20T19:41:35.297 INFO:teuthology.orchestra.run.smithi116.stdout: 5 165802 10.65 MB/sec execute 265 sec latency 58.154 ms 2024-09-20T19:41:36.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 166148 10.63 MB/sec execute 266 sec latency 44.055 ms 2024-09-20T19:41:36.672 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:36.672 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:36.672 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:36 smithi116 ceph-mon[63578]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 412 op/s; 215/1698 objects degraded (12.662%) 2024-09-20T19:41:36.672 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:36.672 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:36.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:36 smithi118 ceph-mon[49197]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 412 op/s; 215/1698 objects degraded (12.662%) 2024-09-20T19:41:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:37.176 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:36 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[68342]: 2024-09-20T19:41:36.989+0000 7f1bd157d640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-20T19:41:37.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 166358 10.63 MB/sec execute 267 sec latency 57.288 ms 2024-09-20T19:41:37.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:37 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 215/1698 objects degraded (12.662%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:37.967 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:37 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 215/1698 objects degraded (12.662%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:38.184 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:37 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[68342]: 2024-09-20T19:41:37.930+0000 7f1bd157d640 -1 osd.0 73 log_to_monitors true 2024-09-20T19:41:38.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 166600 10.61 MB/sec execute 268 sec latency 89.718 ms 2024-09-20T19:41:38.702 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:41:38 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[68342]: 2024-09-20T19:41:38.695+0000 7f1bc9327640 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:41:38.702 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:38 smithi116 ceph-mon[63578]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 412 op/s; 215/1698 objects degraded (12.662%) 2024-09-20T19:41:38.703 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:38 smithi116 ceph-mon[63578]: from='osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-20T19:41:38.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:38 smithi118 ceph-mon[49197]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 412 op/s; 215/1698 objects degraded (12.662%) 2024-09-20T19:41:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:38 smithi118 ceph-mon[49197]: from='osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-20T19:41:39.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 166885 10.61 MB/sec execute 269 sec latency 52.238 ms 2024-09-20T19:41:39.725 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:39 smithi116 ceph-mon[63578]: from='osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T19:41:39.725 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:39 smithi116 ceph-mon[63578]: osdmap e76: 8 total, 7 up, 8 in 2024-09-20T19:41:39.725 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:39 smithi116 ceph-mon[63578]: from='osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:41:39.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:39 smithi118 ceph-mon[49197]: from='osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T19:41:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:39 smithi118 ceph-mon[49197]: osdmap e76: 8 total, 7 up, 8 in 2024-09-20T19:41:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:39 smithi118 ceph-mon[49197]: from='osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:41:40.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 167001 10.58 MB/sec execute 270 sec latency 663.984 ms 2024-09-20T19:41:40.901 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:40 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:41:40.901 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:40 smithi116 ceph-mon[63578]: osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207] boot 2024-09-20T19:41:40.901 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:40 smithi116 ceph-mon[63578]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T19:41:40.901 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:40 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T19:41:40.902 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:40 smithi116 ceph-mon[63578]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 543 op/s; 217/1665 objects degraded (13.033%) 2024-09-20T19:41:40.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:40 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:41:40.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:40 smithi118 ceph-mon[49197]: osd.0 [v2:172.21.15.116:6802/2954864207,v1:172.21.15.116:6803/2954864207] boot 2024-09-20T19:41:40.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:40 smithi118 ceph-mon[49197]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T19:41:40.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:40 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T19:41:40.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:40 smithi118 ceph-mon[49197]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 543 op/s; 217/1665 objects degraded (13.033%) 2024-09-20T19:41:41.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 167116 10.54 MB/sec execute 271 sec latency 1664.010 ms 2024-09-20T19:41:41.668 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:41:41.857 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:41 smithi116 ceph-mon[63578]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T19:41:41.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:41 smithi118 ceph-mon[49197]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T19:41:42.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 167210 10.52 MB/sec execute 272 sec latency 2267.808 ms 2024-09-20T19:41:42.580 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:41:42.580 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (4m) 6s ago 13m 15.6M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (54s) 6s ago 12m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (48s) 43s ago 12m 37.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (2m) 6s ago 12m 46.2M - dad864ee21e9 8a163a77164a 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (7m) 6s ago 7m 37.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (7m) 43s ago 7m 14.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (3m) 6s ago 14m 492M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (2m) 43s ago 12m 448M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (2m) 6s ago 14m 46.8M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (104s) 43s ago 12m 38.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (6m) 6s ago 6m 283M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (6m) 43s ago 6m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (3m) 6s ago 12m 9705k - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (3m) 43s ago 12m 10.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:41:42.581 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (15s) 6s ago 11m 12.1M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (11m) 6s ago 11m 481M 1936M 17.2.0 e1d6a67b021e 9c73ce47d87e 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (10m) 6s ago 10m 364M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (10m) 6s ago 10m 513M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (9m) 43s ago 9m 517M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (9m) 43s ago 9m 318M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (9m) 43s ago 9m 512M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (8m) 43s ago 8m 383M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:41:42.582 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (2m) 6s ago 12m 48.7M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.6 MiB/s wr, 518 op/s; 217/1665 objects degraded (13.033%) 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 217/1665 objects degraded (13.033%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:41:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:42.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.6 MiB/s wr, 518 op/s; 217/1665 objects degraded (13.033%) 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 217/1665 objects degraded (13.033%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:42.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:43.206 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:41:43.206 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:41:43.206 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:41:43.206 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:43.206 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 5 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:41:43.207 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:41:43.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 167510 10.52 MB/sec execute 273 sec latency 49.779 ms 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:41:43.719 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "7/23 daemons upgraded", 2024-09-20T19:41:43.720 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:41:43.720 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:41:43.720 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:41:43.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:43.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:41:43.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:43.850 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:43.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:43.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T19:41:43.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T19:41:43.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-20T19:41:43.851 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:43 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2327183557' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:43.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='client.44173 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-20T19:41:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:43 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2327183557' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:41:44.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 167707 10.49 MB/sec execute 274 sec latency 444.271 ms 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); Degraded data redundancy: 217/1665 objects degraded (13.033%), 29 pgs degraded; 5 mgr modules have recently crashed 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 217/1665 objects degraded (13.033%), 29 pgs degraded 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-20T19:41:44.413 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-20T19:41:44.414 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:41:44.415 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:41:44.416 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:41:44.416 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:41:44.416 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:41:44.416 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:41:44.955 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:44 smithi116 ceph-mon[63578]: pgmap v72: 97 pgs: 2 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.1 MiB/s wr, 272 op/s; 163/1668 objects degraded (9.772%); 23 KiB/s, 23 objects/s recovering 2024-09-20T19:41:44.955 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:44 smithi116 ceph-mon[63578]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T19:41:44.956 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:44 smithi116 ceph-mon[63578]: from='client.44181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:44.956 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:44 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1397882053' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:41:44.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:44 smithi118 ceph-mon[49197]: pgmap v72: 97 pgs: 2 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.1 MiB/s wr, 272 op/s; 163/1668 objects degraded (9.772%); 23 KiB/s, 23 objects/s recovering 2024-09-20T19:41:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:44 smithi118 ceph-mon[49197]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T19:41:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:44 smithi118 ceph-mon[49197]: from='client.44181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:41:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:44 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1397882053' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:41:45.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 167953 10.47 MB/sec execute 275 sec latency 858.030 ms 2024-09-20T19:41:45.723 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:45 smithi116 ceph-mon[63578]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T19:41:45.723 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:45 smithi116 ceph-mon[63578]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T19:41:45.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:45 smithi118 ceph-mon[49197]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T19:41:45.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:45 smithi118 ceph-mon[49197]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T19:41:46.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 168225 10.45 MB/sec execute 276 sec latency 68.190 ms 2024-09-20T19:41:46.758 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:46 smithi116 ceph-mon[63578]: pgmap v75: 97 pgs: 2 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.1 MiB/s wr, 273 op/s; 163/1668 objects degraded (9.772%); 23 KiB/s, 23 objects/s recovering 2024-09-20T19:41:46.758 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:46 smithi116 ceph-mon[63578]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T19:41:46.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:46 smithi118 ceph-mon[49197]: pgmap v75: 97 pgs: 2 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.1 MiB/s wr, 273 op/s; 163/1668 objects degraded (9.772%); 23 KiB/s, 23 objects/s recovering 2024-09-20T19:41:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:46 smithi118 ceph-mon[49197]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T19:41:47.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 168548 10.42 MB/sec execute 277 sec latency 487.499 ms 2024-09-20T19:41:47.771 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:47 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 163/1668 objects degraded (9.772%), 8 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:47.771 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:47 smithi116 ceph-mon[63578]: osdmap e84: 8 total, 8 up, 8 in 2024-09-20T19:41:48.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:47 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 163/1668 objects degraded (9.772%), 8 pgs degraded (PG_DEGRADED) 2024-09-20T19:41:48.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:47 smithi118 ceph-mon[49197]: osdmap e84: 8 total, 8 up, 8 in 2024-09-20T19:41:48.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 168832 10.43 MB/sec execute 278 sec latency 517.005 ms 2024-09-20T19:41:48.806 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:48 smithi116 ceph-mon[63578]: pgmap v78: 97 pgs: 2 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 163/1668 objects degraded (9.772%) 2024-09-20T19:41:49.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:48 smithi118 ceph-mon[49197]: pgmap v78: 97 pgs: 2 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 163/1668 objects degraded (9.772%) 2024-09-20T19:41:49.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 169162 10.42 MB/sec execute 279 sec latency 65.775 ms 2024-09-20T19:41:49.828 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:49 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 163/1668 objects degraded (9.772%), 8 pgs degraded) 2024-09-20T19:41:50.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:49 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 163/1668 objects degraded (9.772%), 8 pgs degraded) 2024-09-20T19:41:50.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 169694 10.44 MB/sec execute 280 sec latency 32.069 ms 2024-09-20T19:41:50.846 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:50 smithi116 ceph-mon[63578]: pgmap v80: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 425 op/s; 0 B/s, 33 objects/s recovering 2024-09-20T19:41:51.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:50 smithi118 ceph-mon[49197]: pgmap v80: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 425 op/s; 0 B/s, 33 objects/s recovering 2024-09-20T19:41:51.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 170083 10.46 MB/sec execute 281 sec latency 37.022 ms 2024-09-20T19:41:52.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 170426 10.46 MB/sec execute 282 sec latency 31.633 ms 2024-09-20T19:41:52.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:52 smithi116 ceph-mon[63578]: pgmap v81: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.1 MiB/s wr, 353 op/s; 0 B/s, 27 objects/s recovering 2024-09-20T19:41:53.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:52 smithi118 ceph-mon[49197]: pgmap v81: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.1 MiB/s wr, 353 op/s; 0 B/s, 27 objects/s recovering 2024-09-20T19:41:53.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 170786 10.46 MB/sec execute 283 sec latency 34.299 ms 2024-09-20T19:41:54.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 171049 10.45 MB/sec execute 284 sec latency 69.786 ms 2024-09-20T19:41:54.901 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:54 smithi116 ceph-mon[63578]: pgmap v82: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.1 MiB/s wr, 600 op/s; 0 B/s, 20 objects/s recovering 2024-09-20T19:41:54.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:54 smithi118 ceph-mon[49197]: pgmap v82: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.1 MiB/s wr, 600 op/s; 0 B/s, 20 objects/s recovering 2024-09-20T19:41:55.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 171319 10.42 MB/sec execute 285 sec latency 73.592 ms 2024-09-20T19:41:56.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 171691 10.41 MB/sec execute 286 sec latency 41.673 ms 2024-09-20T19:41:56.936 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:56 smithi116 ceph-mon[63578]: pgmap v83: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 533 op/s; 0 B/s, 18 objects/s recovering 2024-09-20T19:41:57.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:56 smithi118 ceph-mon[49197]: pgmap v83: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 533 op/s; 0 B/s, 18 objects/s recovering 2024-09-20T19:41:57.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 172025 10.39 MB/sec execute 287 sec latency 51.223 ms 2024-09-20T19:41:58.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 172359 10.39 MB/sec execute 288 sec latency 38.785 ms 2024-09-20T19:41:58.718 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:58.718 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:58.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:58.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:58 smithi116 ceph-mon[63578]: pgmap v84: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 479 op/s; 0 B/s, 16 objects/s recovering 2024-09-20T19:41:58.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T19:41:58.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:58 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T19:41:58.719 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:41:58 smithi116 ceph-mon[63578]: Upgrade: osd.1 is safe to restart 2024-09-20T19:41:58.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:41:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:41:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:58 smithi118 ceph-mon[49197]: pgmap v84: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 479 op/s; 0 B/s, 16 objects/s recovering 2024-09-20T19:41:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T19:41:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:58 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T19:41:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:41:58 smithi118 ceph-mon[49197]: Upgrade: osd.1 is safe to restart 2024-09-20T19:41:59.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 172798 10.41 MB/sec execute 289 sec latency 57.253 ms 2024-09-20T19:42:00.298 INFO:teuthology.orchestra.run.smithi116.stdout: 5 173224 10.41 MB/sec execute 290 sec latency 43.168 ms 2024-09-20T19:42:01.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 173511 10.41 MB/sec execute 291 sec latency 67.614 ms 2024-09-20T19:42:01.509 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:01 smithi116 ceph-mon[63578]: pgmap v85: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 609 op/s; 0 B/s, 13 objects/s recovering 2024-09-20T19:42:01.509 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:01 smithi116 ceph-mon[63578]: Upgrade: Updating osd.1 2024-09-20T19:42:01.509 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:01.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T19:42:01.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:42:01.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:01 smithi116 ceph-mon[63578]: Deploying daemon osd.1 on smithi116 2024-09-20T19:42:01.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:01 smithi118 ceph-mon[49197]: pgmap v85: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 609 op/s; 0 B/s, 13 objects/s recovering 2024-09-20T19:42:01.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:01 smithi118 ceph-mon[49197]: Upgrade: Updating osd.1 2024-09-20T19:42:01.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:01.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T19:42:01.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:42:01.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:01 smithi118 ceph-mon[49197]: Deploying daemon osd.1 on smithi116 2024-09-20T19:42:02.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 173771 10.41 MB/sec execute 292 sec latency 71.832 ms 2024-09-20T19:42:03.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 174055 10.40 MB/sec execute 293 sec latency 64.782 ms 2024-09-20T19:42:03.574 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:03 smithi116 ceph-mon[63578]: pgmap v86: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 431 op/s 2024-09-20T19:42:03.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:03 smithi118 ceph-mon[49197]: pgmap v86: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 431 op/s 2024-09-20T19:42:04.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 174359 10.40 MB/sec execute 294 sec latency 47.833 ms 2024-09-20T19:42:05.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 174563 10.38 MB/sec execute 295 sec latency 72.117 ms 2024-09-20T19:42:05.405 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:05 smithi116 systemd[1]: Stopping Ceph osd.1 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:42:05.656 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:05 smithi116 ceph-mon[63578]: pgmap v87: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-09-20T19:42:05.657 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:05 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[38211]: 2024-09-20T19:42:05.534+0000 7fcdcd4bd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:42:05.657 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:05 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[38211]: 2024-09-20T19:42:05.534+0000 7fcdcd4bd700 -1 osd.1 84 *** Got signal Terminated *** 2024-09-20T19:42:05.657 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:05 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[38211]: 2024-09-20T19:42:05.534+0000 7fcdcd4bd700 -1 osd.1 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:42:05.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:05 smithi118 ceph-mon[49197]: pgmap v87: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-09-20T19:42:06.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 174858 10.36 MB/sec execute 296 sec latency 72.616 ms 2024-09-20T19:42:06.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:06 smithi118 ceph-mon[49197]: osd.1 marked itself down and dead 2024-09-20T19:42:06.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:06 smithi116 ceph-mon[63578]: osd.1 marked itself down and dead 2024-09-20T19:42:07.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 174916 10.32 MB/sec execute 297 sec latency 872.293 ms 2024-09-20T19:42:07.372 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:07 smithi116 podman[70227]: 2024-09-20 19:42:07.111603614 +0000 UTC m=+1.691189604 container died 9c73ce47d87eb6be94588db8ad14ae43f55bede60c82eb693ed261aafc34c533 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, architecture=x86_64, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.buildah.version=1.19.8, name=centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True) 2024-09-20T19:42:07.627 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:07 smithi116 ceph-mon[63578]: pgmap v88: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-20T19:42:07.627 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:07 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:42:07.627 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:07 smithi116 ceph-mon[63578]: osdmap e85: 8 total, 7 up, 8 in 2024-09-20T19:42:07.627 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:07 smithi116 podman[70227]: 2024-09-20 19:42:07.459440694 +0000 UTC m=+2.039026685 container cleanup 9c73ce47d87eb6be94588db8ad14ae43f55bede60c82eb693ed261aafc34c533 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, architecture=x86_64, ceph=True, vendor=Red Hat, Inc., distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-20T19:42:07.627 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:07 smithi116 bash[70227]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1 2024-09-20T19:42:07.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:07 smithi118 ceph-mon[49197]: pgmap v88: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-20T19:42:07.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:07 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:42:07.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:07 smithi118 ceph-mon[49197]: osdmap e85: 8 total, 7 up, 8 in 2024-09-20T19:42:07.879 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:07 smithi116 podman[70243]: 2024-09-20 19:42:07.710354098 +0000 UTC m=+0.596175962 container remove 9c73ce47d87eb6be94588db8ad14ae43f55bede60c82eb693ed261aafc34c533 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, io.openshift.expose-services=, GIT_BRANCH=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., version=8, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-20T19:42:08.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 175351 10.32 MB/sec execute 298 sec latency 1042.628 ms 2024-09-20T19:42:08.392 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:08 smithi116 podman[70329]: 2024-09-20 19:42:08.192598238 +0000 UTC m=+0.023294859 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:42:08.649 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:08 smithi116 ceph-mon[63578]: osdmap e86: 8 total, 7 up, 8 in 2024-09-20T19:42:08.650 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:08 smithi116 podman[70329]: 2024-09-20 19:42:08.431090316 +0000 UTC m=+0.261786923 container create bd81a9c7a3c455e81f759d815e5e0bb8d6ae65331b67499310d69459985b81ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True) 2024-09-20T19:42:08.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:08 smithi118 ceph-mon[49197]: osdmap e86: 8 total, 7 up, 8 in 2024-09-20T19:42:08.900 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:08 smithi116 podman[70329]: 2024-09-20 19:42:08.808195212 +0000 UTC m=+0.638891828 container init bd81a9c7a3c455e81f759d815e5e0bb8d6ae65331b67499310d69459985b81ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-20T19:42:08.901 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:08 smithi116 podman[70329]: 2024-09-20 19:42:08.8202879 +0000 UTC m=+0.650984520 container start bd81a9c7a3c455e81f759d815e5e0bb8d6ae65331b67499310d69459985b81ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-20T19:42:09.153 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:08 smithi116 podman[70329]: 2024-09-20 19:42:08.925451531 +0000 UTC m=+0.756148163 container attach bd81a9c7a3c455e81f759d815e5e0bb8d6ae65331b67499310d69459985b81ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-20T19:42:09.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 175840 10.33 MB/sec execute 299 sec latency 65.673 ms 2024-09-20T19:42:09.408 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:09 smithi116 podman[70329]: 2024-09-20 19:42:09.171018613 +0000 UTC m=+1.001715229 container died bd81a9c7a3c455e81f759d815e5e0bb8d6ae65331b67499310d69459985b81ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-20T19:42:09.658 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:09 smithi116 ceph-mon[63578]: pgmap v91: 97 pgs: 14 stale+active+clean, 83 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.5 MiB/s wr, 302 op/s 2024-09-20T19:42:09.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:09 smithi118 ceph-mon[49197]: pgmap v91: 97 pgs: 14 stale+active+clean, 83 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.5 MiB/s wr, 302 op/s 2024-09-20T19:42:10.299 INFO:teuthology.orchestra.run.smithi116.stdout: 5 176378 10.35 MB/sec execute 300 sec latency 33.411 ms 2024-09-20T19:42:10.422 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:10 smithi116 podman[70355]: 2024-09-20 19:42:10.406834084 +0000 UTC m=+1.218850638 container remove bd81a9c7a3c455e81f759d815e5e0bb8d6ae65331b67499310d69459985b81ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T19:42:10.673 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:10 smithi116 ceph-mon[63578]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 572 op/s; 177/1620 objects degraded (10.926%) 2024-09-20T19:42:10.674 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:10 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 177/1620 objects degraded (10.926%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:10.674 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:10 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-20T19:42:10.674 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:10 smithi116 systemd[1]: Stopped Ceph osd.1 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:42:10.674 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:10 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.1.service: Consumed 2min 54.712s CPU time. 2024-09-20T19:42:10.822 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:10 smithi116 systemd[1]: Starting Ceph osd.1 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:42:10.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:10 smithi118 ceph-mon[49197]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 572 op/s; 177/1620 objects degraded (10.926%) 2024-09-20T19:42:10.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:10 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 177/1620 objects degraded (10.926%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:11.185 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 podman[70451]: 2024-09-20 19:42:10.983048579 +0000 UTC m=+0.026547856 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:42:11.185 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 podman[70451]: 2024-09-20 19:42:11.119164369 +0000 UTC m=+0.162663629 container create 83ecda7bd3e58f93db54f2d1329bddaf5909d611dc8c73b9629b16c9cc7566ec (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0) 2024-09-20T19:42:11.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 176847 10.36 MB/sec execute 301 sec latency 36.435 ms 2024-09-20T19:42:11.707 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 podman[70451]: 2024-09-20 19:42:11.464267074 +0000 UTC m=+0.507766337 container init 83ecda7bd3e58f93db54f2d1329bddaf5909d611dc8c73b9629b16c9cc7566ec (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:42:11.707 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 podman[70451]: 2024-09-20 19:42:11.478334708 +0000 UTC m=+0.521833969 container start 83ecda7bd3e58f93db54f2d1329bddaf5909d611dc8c73b9629b16c9cc7566ec (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-20T19:42:11.707 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 podman[70451]: 2024-09-20 19:42:11.62915949 +0000 UTC m=+0.672658752 container attach 83ecda7bd3e58f93db54f2d1329bddaf5909d611dc8c73b9629b16c9cc7566ec (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-20T19:42:11.707 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:11.707 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 bash[70451]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:11.707 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:11.707 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:11 smithi116 bash[70451]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:12.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 177163 10.37 MB/sec execute 302 sec latency 43.090 ms 2024-09-20T19:42:12.984 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:12 smithi116 ceph-mon[63578]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 MiB/s wr, 269 op/s; 177/1620 objects degraded (10.926%) 2024-09-20T19:42:12.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:12.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:12.985 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:12 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T19:42:12.985 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:12 smithi116 bash[70451]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T19:42:12.985 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:12 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T19:42:12.985 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:12 smithi116 bash[70451]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T19:42:13.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:12 smithi118 ceph-mon[49197]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 MiB/s wr, 269 op/s; 177/1620 objects degraded (10.926%) 2024-09-20T19:42:13.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:13.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:13.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 177517 10.37 MB/sec execute 303 sec latency 44.064 ms 2024-09-20T19:42:13.494 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T19:42:13.494 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 bash[70451]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T19:42:13.494 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T19:42:13.494 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 bash[70451]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T19:42:13.494 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-20T19:42:13.494 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 bash[70451]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-20T19:42:13.494 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T19:42:13.495 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 bash[70451]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T19:42:13.495 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate[70463]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-20T19:42:13.495 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 bash[70451]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-20T19:42:13.495 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:13 smithi116 podman[70451]: 2024-09-20 19:42:13.336004293 +0000 UTC m=+2.379503552 container died 83ecda7bd3e58f93db54f2d1329bddaf5909d611dc8c73b9629b16c9cc7566ec (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T19:42:14.262 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 podman[70621]: 2024-09-20 19:42:14.141288836 +0000 UTC m=+0.797054109 container remove 83ecda7bd3e58f93db54f2d1329bddaf5909d611dc8c73b9629b16c9cc7566ec (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, ceph=True) 2024-09-20T19:42:14.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 177881 10.37 MB/sec execute 304 sec latency 58.221 ms 2024-09-20T19:42:14.527 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 podman[70678]: 2024-09-20 19:42:14.29713915 +0000 UTC m=+0.026087209 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:42:14.527 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 podman[70678]: 2024-09-20 19:42:14.416105802 +0000 UTC m=+0.145053846 container create d48d1daf3d38232800ec75a15c4f2b53d5f0a45de1f7043b8a141faf28e23bc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:42:14.787 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:14 smithi116 ceph-mon[63578]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 630 op/s; 189/1716 objects degraded (11.014%) 2024-09-20T19:42:14.788 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 podman[70678]: 2024-09-20 19:42:14.709916667 +0000 UTC m=+0.438864716 container init d48d1daf3d38232800ec75a15c4f2b53d5f0a45de1f7043b8a141faf28e23bc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:42:14.788 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 podman[70678]: 2024-09-20 19:42:14.728703753 +0000 UTC m=+0.457651801 container start d48d1daf3d38232800ec75a15c4f2b53d5f0a45de1f7043b8a141faf28e23bc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-20T19:42:14.788 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 ceph-osd[70717]: -- 172.21.15.116:0/3495746446 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f7d81441e0 con 0x55f7d8134000 2024-09-20T19:42:14.938 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:42:15.050 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 bash[70678]: d48d1daf3d38232800ec75a15c4f2b53d5f0a45de1f7043b8a141faf28e23bc9 2024-09-20T19:42:15.051 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:14 smithi116 systemd[1]: Started Ceph osd.1 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:42:15.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:14 smithi118 ceph-mon[49197]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 630 op/s; 189/1716 objects degraded (11.014%) 2024-09-20T19:42:15.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 178108 10.34 MB/sec execute 305 sec latency 53.801 ms 2024-09-20T19:42:15.892 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:42:15.892 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (4m) 40s ago 13m 15.6M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:42:15.892 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (87s) 40s ago 13m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:42:15.892 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (81s) 76s ago 13m 37.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:42:15.892 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (3m) 40s ago 13m 46.2M - dad864ee21e9 8a163a77164a 2024-09-20T19:42:15.892 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (7m) 40s ago 7m 37.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (7m) 76s ago 7m 14.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (3m) 40s ago 14m 492M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (3m) 76s ago 12m 448M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (2m) 40s ago 14m 46.8M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (2m) 76s ago 12m 38.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (7m) 40s ago 7m 283M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (7m) 76s ago 7m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (4m) 40s ago 13m 9705k - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (4m) 76s ago 12m 10.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (49s) 40s ago 12m 12.1M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 starting - - - 1936M 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (11m) 40s ago 11m 364M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:42:15.893 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (10m) 40s ago 10m 513M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:42:15.894 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (10m) 76s ago 10m 517M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:42:15.894 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (9m) 76s ago 9m 318M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:42:15.894 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (9m) 76s ago 9m 512M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:42:15.894 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (9m) 76s ago 9m 383M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:42:15.894 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (3m) 40s ago 12m 48.7M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:42:16.054 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:16 smithi116 ceph-mon[63578]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:16.054 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:16.054 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:16.055 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:42:16.055 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:16 smithi116 ceph-mon[63578]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:16.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 178446 10.33 MB/sec execute 306 sec latency 64.720 ms 2024-09-20T19:42:16.306 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:16 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[70713]: 2024-09-20T19:42:16.177+0000 7f5e17118640 -1 Falling back to public interface 2024-09-20T19:42:16.386 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:42:16.386 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:16.387 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:42:16.388 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-20T19:42:16.388 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 5 2024-09-20T19:42:16.388 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:42:16.388 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:42:16.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:16 smithi118 ceph-mon[49197]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:42:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:16 smithi118 ceph-mon[49197]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:16.812 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:42:16.812 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "7/23 daemons upgraded", 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:42:16.813 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:42:17.093 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:17 smithi116 ceph-mon[63578]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 545 op/s; 189/1716 objects degraded (11.014%) 2024-09-20T19:42:17.093 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:17 smithi116 ceph-mon[63578]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:17.093 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:17 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/840393289' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:17.287 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 osds down; Degraded data redundancy: 189/1716 objects degraded (11.014%), 24 pgs degraded; 5 mgr modules have recently crashed 2024-09-20T19:42:17.287 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:42:17.287 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:42:17.287 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: osd.1 (root=default,host=smithi116) is down 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 189/1716 objects degraded (11.014%), 24 pgs degraded 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-20T19:42:17.288 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:42:17.289 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:42:17.290 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:42:17.290 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:42:17.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 178683 10.32 MB/sec execute 307 sec latency 121.578 ms 2024-09-20T19:42:17.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:17 smithi118 ceph-mon[49197]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 545 op/s; 189/1716 objects degraded (11.014%) 2024-09-20T19:42:17.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:17 smithi118 ceph-mon[49197]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:17.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:17 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/840393289' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:18.119 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:18 smithi116 ceph-mon[63578]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:18.120 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:18 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2894099670' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:42:18.120 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:18 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 189/1716 objects degraded (11.014%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:18.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 178965 10.30 MB/sec execute 308 sec latency 62.342 ms 2024-09-20T19:42:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:18 smithi118 ceph-mon[49197]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:18 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2894099670' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:42:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:18 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 189/1716 objects degraded (11.014%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:19.150 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:19 smithi116 ceph-mon[63578]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 189/1716 objects degraded (11.014%) 2024-09-20T19:42:19.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 179285 10.29 MB/sec execute 309 sec latency 86.492 ms 2024-09-20T19:42:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:19 smithi118 ceph-mon[49197]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 189/1716 objects degraded (11.014%) 2024-09-20T19:42:20.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 179557 10.29 MB/sec execute 310 sec latency 54.786 ms 2024-09-20T19:42:21.204 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:21 smithi116 ceph-mon[63578]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 191/1653 objects degraded (11.555%) 2024-09-20T19:42:21.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 180033 10.30 MB/sec execute 311 sec latency 52.155 ms 2024-09-20T19:42:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:21 smithi118 ceph-mon[49197]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 191/1653 objects degraded (11.555%) 2024-09-20T19:42:22.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 180405 10.30 MB/sec execute 312 sec latency 44.845 ms 2024-09-20T19:42:23.239 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:23 smithi116 ceph-mon[63578]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 191/1653 objects degraded (11.555%) 2024-09-20T19:42:23.240 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:23 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 191/1653 objects degraded (11.555%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:23.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 180701 10.31 MB/sec execute 313 sec latency 58.078 ms 2024-09-20T19:42:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:23 smithi118 ceph-mon[49197]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 191/1653 objects degraded (11.555%) 2024-09-20T19:42:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:23 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 191/1653 objects degraded (11.555%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:23.500 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:23 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[70713]: 2024-09-20T19:42:23.383+0000 7f5e17118640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-20T19:42:24.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 180930 10.30 MB/sec execute 314 sec latency 65.159 ms 2024-09-20T19:42:24.537 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:24 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[70713]: 2024-09-20T19:42:24.377+0000 7f5e17118640 -1 osd.1 84 log_to_monitors true 2024-09-20T19:42:25.295 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:42:25 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[70713]: 2024-09-20T19:42:25.167+0000 7f5e0eec2640 -1 osd.1 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:42:25.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:25 smithi116 ceph-mon[63578]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 629 op/s; 178/1680 objects degraded (10.595%) 2024-09-20T19:42:25.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:25.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:25.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:25 smithi116 ceph-mon[63578]: from='osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-20T19:42:25.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:25.296 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:25.300 INFO:teuthology.orchestra.run.smithi116.stdout: 5 181183 10.29 MB/sec execute 315 sec latency 70.278 ms 2024-09-20T19:42:25.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:25 smithi118 ceph-mon[49197]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.6 MiB/s wr, 629 op/s; 178/1680 objects degraded (10.595%) 2024-09-20T19:42:25.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:25.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:25.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:25 smithi118 ceph-mon[49197]: from='osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-20T19:42:25.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:25.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:26.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 181386 10.27 MB/sec execute 316 sec latency 212.029 ms 2024-09-20T19:42:26.355 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:26 smithi116 ceph-mon[63578]: from='osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T19:42:26.355 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:26 smithi116 ceph-mon[63578]: osdmap e87: 8 total, 7 up, 8 in 2024-09-20T19:42:26.355 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:26 smithi116 ceph-mon[63578]: from='osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:42:26.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:26 smithi118 ceph-mon[49197]: from='osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T19:42:26.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:26 smithi118 ceph-mon[49197]: osdmap e87: 8 total, 7 up, 8 in 2024-09-20T19:42:26.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:26 smithi118 ceph-mon[49197]: from='osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:42:27.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 181386 10.24 MB/sec execute 317 sec latency 1212.058 ms 2024-09-20T19:42:27.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:27 smithi116 ceph-mon[63578]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.5 MiB/s wr, 466 op/s; 178/1680 objects degraded (10.595%) 2024-09-20T19:42:27.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:27 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:42:27.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:27 smithi116 ceph-mon[63578]: osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223] boot 2024-09-20T19:42:27.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:27 smithi116 ceph-mon[63578]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T19:42:27.363 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:27 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T19:42:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:27 smithi118 ceph-mon[49197]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.5 MiB/s wr, 466 op/s; 178/1680 objects degraded (10.595%) 2024-09-20T19:42:27.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:27 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:42:27.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:27 smithi118 ceph-mon[49197]: osd.1 [v2:172.21.15.116:6810/2428292223,v1:172.21.15.116:6811/2428292223] boot 2024-09-20T19:42:27.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:27 smithi118 ceph-mon[49197]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T19:42:27.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:27 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T19:42:28.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 181386 10.21 MB/sec execute 318 sec latency 2212.117 ms 2024-09-20T19:42:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:28 smithi118 ceph-mon[49197]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T19:42:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:28 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 178/1680 objects degraded (10.595%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:28.559 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:28 smithi116 ceph-mon[63578]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T19:42:28.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:28 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 178/1680 objects degraded (10.595%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:28.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:28.560 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:29.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 181560 10.18 MB/sec execute 319 sec latency 2626.613 ms 2024-09-20T19:42:29.331 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:29 smithi116 ceph-mon[63578]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 178/1680 objects degraded (10.595%) 2024-09-20T19:42:29.331 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:29 smithi116 ceph-mon[63578]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T19:42:29.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:29 smithi118 ceph-mon[49197]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 178/1680 objects degraded (10.595%) 2024-09-20T19:42:29.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:29 smithi118 ceph-mon[49197]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T19:42:30.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 181758 10.17 MB/sec execute 320 sec latency 73.022 ms 2024-09-20T19:42:30.615 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T19:42:30.615 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: pgmap v107: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 8 remapped+peering, 5 active+undersized, 4 active+undersized+degraded, 77 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.0 MiB/s wr, 262 op/s; 60/1626 objects degraded (3.690%); 980 KiB/s, 14 objects/s recovering 2024-09-20T19:42:30.615 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:30.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:30.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:42:30.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:42:30.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:30.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:30 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: pgmap v107: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 8 remapped+peering, 5 active+undersized, 4 active+undersized+degraded, 77 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.0 MiB/s wr, 262 op/s; 60/1626 objects degraded (3.690%); 980 KiB/s, 14 objects/s recovering 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:30.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:30 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:31.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 182026 10.16 MB/sec execute 321 sec latency 247.304 ms 2024-09-20T19:42:31.676 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: osdmap e92: 8 total, 8 up, 8 in 2024-09-20T19:42:31.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:42:31.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:31.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:31.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:31.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T19:42:31.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T19:42:31.677 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:31 smithi116 ceph-mon[63578]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: osdmap e92: 8 total, 8 up, 8 in 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T19:42:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:31 smithi118 ceph-mon[49197]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-20T19:42:32.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 182315 10.14 MB/sec execute 322 sec latency 726.103 ms 2024-09-20T19:42:32.705 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:32 smithi116 ceph-mon[63578]: osdmap e93: 8 total, 8 up, 8 in 2024-09-20T19:42:32.706 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:32 smithi116 ceph-mon[63578]: pgmap v110: 97 pgs: 2 peering, 2 activating+undersized+degraded+remapped, 7 remapped+peering, 86 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.1 MiB/s wr, 269 op/s; 45/1623 objects degraded (2.773%); 1.1 MiB/s, 63 keys/s, 15 objects/s recovering 2024-09-20T19:42:32.706 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:32 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 60/1626 objects degraded (3.690%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:32.706 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:32 smithi116 ceph-mon[63578]: osdmap e94: 8 total, 8 up, 8 in 2024-09-20T19:42:32.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:32 smithi118 ceph-mon[49197]: osdmap e93: 8 total, 8 up, 8 in 2024-09-20T19:42:32.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:32 smithi118 ceph-mon[49197]: pgmap v110: 97 pgs: 2 peering, 2 activating+undersized+degraded+remapped, 7 remapped+peering, 86 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.1 MiB/s wr, 269 op/s; 45/1623 objects degraded (2.773%); 1.1 MiB/s, 63 keys/s, 15 objects/s recovering 2024-09-20T19:42:32.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:32 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 60/1626 objects degraded (3.690%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:32.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:32 smithi118 ceph-mon[49197]: osdmap e94: 8 total, 8 up, 8 in 2024-09-20T19:42:33.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 182505 10.12 MB/sec execute 323 sec latency 834.051 ms 2024-09-20T19:42:34.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 182865 10.11 MB/sec execute 324 sec latency 863.068 ms 2024-09-20T19:42:34.531 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:34 smithi116 ceph-mon[63578]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T19:42:34.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:34 smithi118 ceph-mon[49197]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T19:42:35.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 183279 10.11 MB/sec execute 325 sec latency 912.402 ms 2024-09-20T19:42:35.538 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:35 smithi116 ceph-mon[63578]: pgmap v113: 97 pgs: 1 active+undersized+remapped, 5 peering, 2 remapped+peering, 89 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.4 MiB/s wr, 447 op/s; 1.2 MiB/s, 63 keys/s, 37 objects/s recovering 2024-09-20T19:42:35.538 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:35 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1623 objects degraded (2.773%), 2 pgs degraded) 2024-09-20T19:42:35.539 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:35 smithi116 ceph-mon[63578]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T19:42:35.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:35 smithi118 ceph-mon[49197]: pgmap v113: 97 pgs: 1 active+undersized+remapped, 5 peering, 2 remapped+peering, 89 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.4 MiB/s wr, 447 op/s; 1.2 MiB/s, 63 keys/s, 37 objects/s recovering 2024-09-20T19:42:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:35 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1623 objects degraded (2.773%), 2 pgs degraded) 2024-09-20T19:42:35.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:35 smithi118 ceph-mon[49197]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T19:42:36.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 183624 10.11 MB/sec execute 326 sec latency 57.843 ms 2024-09-20T19:42:37.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 184182 10.16 MB/sec execute 327 sec latency 52.250 ms 2024-09-20T19:42:37.462 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:37 smithi116 ceph-mon[63578]: pgmap v115: 97 pgs: 3 peering, 94 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.3 MiB/s wr, 445 op/s; 1.7 MiB/s, 50 objects/s recovering 2024-09-20T19:42:37.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:37 smithi118 ceph-mon[49197]: pgmap v115: 97 pgs: 3 peering, 94 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.3 MiB/s wr, 445 op/s; 1.7 MiB/s, 50 objects/s recovering 2024-09-20T19:42:38.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 184858 10.19 MB/sec execute 328 sec latency 21.799 ms 2024-09-20T19:42:39.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 185441 10.19 MB/sec execute 329 sec latency 37.440 ms 2024-09-20T19:42:39.583 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:39 smithi116 ceph-mon[63578]: pgmap v116: 97 pgs: 3 peering, 94 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.3 MiB/s wr, 307 op/s; 1.2 MiB/s, 34 objects/s recovering 2024-09-20T19:42:39.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:39 smithi118 ceph-mon[49197]: pgmap v116: 97 pgs: 3 peering, 94 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.3 MiB/s wr, 307 op/s; 1.2 MiB/s, 34 objects/s recovering 2024-09-20T19:42:40.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 186013 10.19 MB/sec execute 330 sec latency 26.465 ms 2024-09-20T19:42:41.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 186545 10.19 MB/sec execute 331 sec latency 40.539 ms 2024-09-20T19:42:41.601 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:41 smithi116 ceph-mon[63578]: pgmap v117: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.6 MiB/s wr, 667 op/s; 975 KiB/s, 28 objects/s recovering 2024-09-20T19:42:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:41 smithi118 ceph-mon[49197]: pgmap v117: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.6 MiB/s wr, 667 op/s; 975 KiB/s, 28 objects/s recovering 2024-09-20T19:42:42.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 187115 10.21 MB/sec execute 332 sec latency 34.126 ms 2024-09-20T19:42:43.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 187479 10.22 MB/sec execute 333 sec latency 42.476 ms 2024-09-20T19:42:43.646 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:43 smithi116 ceph-mon[63578]: pgmap v118: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.7 MiB/s wr, 401 op/s; 366 KiB/s, 7 objects/s recovering 2024-09-20T19:42:43.646 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:43.647 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:43.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:43 smithi118 ceph-mon[49197]: pgmap v118: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.7 MiB/s wr, 401 op/s; 366 KiB/s, 7 objects/s recovering 2024-09-20T19:42:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:44.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 187801 10.24 MB/sec execute 334 sec latency 56.504 ms 2024-09-20T19:42:45.301 INFO:teuthology.orchestra.run.smithi116.stdout: 5 188098 10.23 MB/sec execute 335 sec latency 55.878 ms 2024-09-20T19:42:45.444 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:45 smithi116 ceph-mon[63578]: pgmap v119: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.2 MiB/s wr, 708 op/s; 304 KiB/s, 6 objects/s recovering 2024-09-20T19:42:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:45 smithi118 ceph-mon[49197]: pgmap v119: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.2 MiB/s wr, 708 op/s; 304 KiB/s, 6 objects/s recovering 2024-09-20T19:42:46.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 188366 10.23 MB/sec execute 336 sec latency 66.595 ms 2024-09-20T19:42:46.459 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:46 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T19:42:46.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:46 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T19:42:47.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 188593 10.21 MB/sec execute 337 sec latency 46.617 ms 2024-09-20T19:42:47.483 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:47 smithi116 ceph-mon[63578]: pgmap v120: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.6 MiB/s wr, 636 op/s; 269 KiB/s, 5 objects/s recovering 2024-09-20T19:42:47.483 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:47 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T19:42:47.483 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:47 smithi116 ceph-mon[63578]: Upgrade: osd.2 is safe to restart 2024-09-20T19:42:47.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:47 smithi118 ceph-mon[49197]: pgmap v120: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.6 MiB/s wr, 636 op/s; 269 KiB/s, 5 objects/s recovering 2024-09-20T19:42:47.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:47 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T19:42:47.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:47 smithi118 ceph-mon[49197]: Upgrade: osd.2 is safe to restart 2024-09-20T19:42:47.848 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:42:48.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 188778 10.18 MB/sec execute 338 sec latency 70.993 ms 2024-09-20T19:42:48.746 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:48.746 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: pgmap v121: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 593 op/s 2024-09-20T19:42:48.746 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:48.746 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: Upgrade: Updating osd.2 2024-09-20T19:42:48.746 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:48.746 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T19:42:48.747 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:42:48.747 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: Deploying daemon osd.2 on smithi116 2024-09-20T19:42:48.747 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:48 smithi116 ceph-mon[63578]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:48.897 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (5m) 24s ago 14m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (2m) 24s ago 14m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (114s) 109s ago 13m 37.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (4m) 24s ago 13m 47.2M - dad864ee21e9 8a163a77164a 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (8m) 24s ago 8m 37.2M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (8m) 109s ago 8m 14.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (4m) 24s ago 15m 494M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (3m) 109s ago 13m 448M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (3m) 24s ago 15m 49.0M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (2m) 109s ago 13m 38.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (7m) 24s ago 7m 284M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:42:48.898 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (7m) 109s ago 7m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (5m) 24s ago 13m 9869k - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (4m) 109s ago 13m 10.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (82s) 24s ago 12m 231M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (34s) 24s ago 12m 12.2M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (11m) 24s ago 11m 413M 1936M 17.2.0 e1d6a67b021e ba938f44579d 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (11m) 24s ago 11m 562M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (10m) 109s ago 10m 517M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (10m) 109s ago 10m 318M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (10m) 109s ago 10m 512M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (9m) 109s ago 9m 383M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:42:48.899 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (3m) 24s ago 13m 50.9M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: pgmap v121: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 593 op/s 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: Upgrade: Updating osd.2 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: Deploying daemon osd.2 on smithi116 2024-09-20T19:42:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:48 smithi118 ceph-mon[49197]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:49.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 189018 10.17 MB/sec execute 339 sec latency 89.951 ms 2024-09-20T19:42:49.507 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:42:49.507 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:42:49.507 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:42:49.507 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 6 2024-09-20T19:42:49.508 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:42:49.509 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:42:49.791 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:49 smithi116 ceph-mon[63578]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:49.791 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:49 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/867748299' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:49.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:49 smithi118 ceph-mon[49197]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:49 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/867748299' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "8/23 daemons upgraded", 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:42:50.127 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:42:50.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 189340 10.16 MB/sec execute 340 sec latency 58.458 ms 2024-09-20T19:42:50.803 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:50 smithi116 ceph-mon[63578]: pgmap v122: 97 pgs: 97 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 750 op/s 2024-09-20T19:42:50.804 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:50 smithi116 ceph-mon[63578]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:42:50.807 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:42:50.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:50 smithi118 ceph-mon[49197]: pgmap v122: 97 pgs: 97 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 750 op/s 2024-09-20T19:42:50.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:50 smithi118 ceph-mon[49197]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:42:51.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 189708 10.15 MB/sec execute 341 sec latency 39.115 ms 2024-09-20T19:42:51.814 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:51 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/69273593' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:42:51.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:51 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/69273593' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:42:52.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 190213 10.16 MB/sec execute 342 sec latency 110.200 ms 2024-09-20T19:42:52.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:52 smithi116 ceph-mon[63578]: pgmap v123: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 504 op/s 2024-09-20T19:42:52.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:52 smithi118 ceph-mon[49197]: pgmap v123: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 504 op/s 2024-09-20T19:42:53.126 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:52 smithi116 systemd[1]: Stopping Ceph osd.2 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:42:53.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 190650 10.16 MB/sec execute 343 sec latency 35.449 ms 2024-09-20T19:42:53.378 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:53 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[41876]: 2024-09-20T19:42:53.326+0000 7f76e5ed1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:42:53.378 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:53 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[41876]: 2024-09-20T19:42:53.326+0000 7f76e5ed1700 -1 osd.2 96 *** Got signal Terminated *** 2024-09-20T19:42:53.378 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:53 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[41876]: 2024-09-20T19:42:53.326+0000 7f76e5ed1700 -1 osd.2 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:42:53.636 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:53 smithi116 ceph-mon[63578]: osd.2 marked itself down and dead 2024-09-20T19:42:53.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:53 smithi118 ceph-mon[49197]: osd.2 marked itself down and dead 2024-09-20T19:42:54.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 190903 10.15 MB/sec execute 344 sec latency 612.552 ms 2024-09-20T19:42:54.619 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:54 smithi116 podman[72744]: 2024-09-20 19:42:54.304760905 +0000 UTC m=+1.129784907 container died ba938f44579db3cf694b14f70c0c91133cc668edc293691724e5ba72d3f1b275 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, ceph=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc.) 2024-09-20T19:42:54.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:54 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:42:54.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:54 smithi116 ceph-mon[63578]: osdmap e97: 8 total, 7 up, 8 in 2024-09-20T19:42:54.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:54 smithi116 ceph-mon[63578]: pgmap v125: 97 pgs: 2 peering, 5 stale+active+clean, 90 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.6 MiB/s wr, 465 op/s 2024-09-20T19:42:54.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:54 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:42:54.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:54 smithi118 ceph-mon[49197]: osdmap e97: 8 total, 7 up, 8 in 2024-09-20T19:42:54.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:54 smithi118 ceph-mon[49197]: pgmap v125: 97 pgs: 2 peering, 5 stale+active+clean, 90 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.6 MiB/s wr, 465 op/s 2024-09-20T19:42:55.129 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:55 smithi116 podman[72744]: 2024-09-20 19:42:55.004941136 +0000 UTC m=+1.829965143 container cleanup ba938f44579db3cf694b14f70c0c91133cc668edc293691724e5ba72d3f1b275 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , distribution-scope=public, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-20T19:42:55.130 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:55 smithi116 bash[72744]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2 2024-09-20T19:42:55.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 191094 10.15 MB/sec execute 345 sec latency 954.466 ms 2024-09-20T19:42:55.380 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:55 smithi116 podman[72763]: 2024-09-20 19:42:55.274137009 +0000 UTC m=+0.968180156 container remove ba938f44579db3cf694b14f70c0c91133cc668edc293691724e5ba72d3f1b275 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, release=754, version=8, architecture=x86_64, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, maintainer=Guillaume Abrioux , ceph=True, vcs-type=git, vendor=Red Hat, Inc.) 2024-09-20T19:42:55.632 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:55 smithi116 ceph-mon[63578]: osdmap e98: 8 total, 7 up, 8 in 2024-09-20T19:42:55.895 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:55 smithi116 podman[72845]: 2024-09-20 19:42:55.679807916 +0000 UTC m=+0.020450402 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:42:55.895 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:55 smithi116 podman[72845]: 2024-09-20 19:42:55.808770013 +0000 UTC m=+0.149412486 container create 3385499a23e41c0bfdd9f989eeef3da11707bb71f36ad81f2c2479cd0bd8fb3b (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-20T19:42:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:55 smithi118 ceph-mon[49197]: osdmap e98: 8 total, 7 up, 8 in 2024-09-20T19:42:56.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 191444 10.16 MB/sec execute 346 sec latency 58.102 ms 2024-09-20T19:42:56.408 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:56 smithi116 podman[72845]: 2024-09-20 19:42:56.151140569 +0000 UTC m=+0.491783039 container init 3385499a23e41c0bfdd9f989eeef3da11707bb71f36ad81f2c2479cd0bd8fb3b (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-20T19:42:56.408 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:56 smithi116 podman[72845]: 2024-09-20 19:42:56.162718125 +0000 UTC m=+0.503360597 container start 3385499a23e41c0bfdd9f989eeef3da11707bb71f36ad81f2c2479cd0bd8fb3b (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-20T19:42:56.409 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:56 smithi116 podman[72845]: 2024-09-20 19:42:56.323033618 +0000 UTC m=+0.663676092 container attach 3385499a23e41c0bfdd9f989eeef3da11707bb71f36ad81f2c2479cd0bd8fb3b (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True) 2024-09-20T19:42:56.673 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:56 smithi116 ceph-mon[63578]: pgmap v127: 97 pgs: 3 active+undersized, 2 peering, 3 stale+active+clean, 3 active+undersized+degraded, 86 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 579 op/s; 40/1665 objects degraded (2.402%) 2024-09-20T19:42:56.673 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:56 smithi116 podman[72845]: 2024-09-20 19:42:56.469152488 +0000 UTC m=+0.809794962 container died 3385499a23e41c0bfdd9f989eeef3da11707bb71f36ad81f2c2479cd0bd8fb3b (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:42:56.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:56 smithi118 ceph-mon[49197]: pgmap v127: 97 pgs: 3 active+undersized, 2 peering, 3 stale+active+clean, 3 active+undersized+degraded, 86 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 579 op/s; 40/1665 objects degraded (2.402%) 2024-09-20T19:42:57.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 191892 10.18 MB/sec execute 347 sec latency 42.580 ms 2024-09-20T19:42:57.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:57 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 40/1665 objects degraded (2.402%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:57.683 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:57.683 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:57.683 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:57 smithi116 podman[72868]: 2024-09-20 19:42:57.615000399 +0000 UTC m=+1.139832212 container remove 3385499a23e41c0bfdd9f989eeef3da11707bb71f36ad81f2c2479cd0bd8fb3b (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-20T19:42:57.934 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:57 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-20T19:42:57.934 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:57 smithi116 systemd[1]: Stopped Ceph osd.2 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:42:57.934 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:57 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.2.service: Consumed 2min 11.950s CPU time. 2024-09-20T19:42:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:57 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 40/1665 objects degraded (2.402%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:42:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:42:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:42:58.184 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 systemd[1]: Starting Ceph osd.2 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:42:58.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 192251 10.17 MB/sec execute 348 sec latency 43.522 ms 2024-09-20T19:42:58.435 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 podman[72964]: 2024-09-20 19:42:58.17982205 +0000 UTC m=+0.022197461 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:42:58.435 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 podman[72964]: 2024-09-20 19:42:58.355281567 +0000 UTC m=+0.197656965 container create 685ab766bea088155784252dd2fd8668c013e70d2603d7540b268866867647ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-20T19:42:58.688 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:42:58 smithi116 ceph-mon[63578]: pgmap v128: 97 pgs: 3 active+undersized, 2 peering, 3 stale+active+clean, 3 active+undersized+degraded, 86 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.8 MiB/s wr, 343 op/s; 40/1665 objects degraded (2.402%) 2024-09-20T19:42:58.688 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 podman[72964]: 2024-09-20 19:42:58.641486362 +0000 UTC m=+0.483861765 container init 685ab766bea088155784252dd2fd8668c013e70d2603d7540b268866867647ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:42:58.689 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 podman[72964]: 2024-09-20 19:42:58.646959346 +0000 UTC m=+0.489334754 container start 685ab766bea088155784252dd2fd8668c013e70d2603d7540b268866867647ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-20T19:42:58.922 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 podman[72964]: 2024-09-20 19:42:58.764491369 +0000 UTC m=+0.606866777 container attach 685ab766bea088155784252dd2fd8668c013e70d2603d7540b268866867647ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:42:58.922 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:58.922 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 bash[72964]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:58.922 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:58.923 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:58 smithi116 bash[72964]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:42:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:42:58 smithi118 ceph-mon[49197]: pgmap v128: 97 pgs: 3 active+undersized, 2 peering, 3 stale+active+clean, 3 active+undersized+degraded, 86 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.8 MiB/s wr, 343 op/s; 40/1665 objects degraded (2.402%) 2024-09-20T19:42:59.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 192643 10.16 MB/sec execute 349 sec latency 44.423 ms 2024-09-20T19:42:59.719 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:59 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T19:42:59.720 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:59 smithi116 bash[72964]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T19:42:59.720 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:59 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T19:42:59.720 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:42:59 smithi116 bash[72964]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T19:43:00.222 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 bash[72964]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 bash[72964]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 bash[72964]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 bash[72964]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate[72980]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 bash[72964]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-20T19:43:00.223 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:00 smithi116 podman[72964]: 2024-09-20 19:43:00.144984666 +0000 UTC m=+1.987360071 container died 685ab766bea088155784252dd2fd8668c013e70d2603d7540b268866867647ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-20T19:43:00.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 193110 10.14 MB/sec execute 350 sec latency 49.999 ms 2024-09-20T19:43:00.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:00 smithi116 ceph-mon[63578]: pgmap v129: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.3 MiB/s wr, 622 op/s; 171/1770 objects degraded (9.661%) 2024-09-20T19:43:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:00 smithi118 ceph-mon[49197]: pgmap v129: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.3 MiB/s wr, 622 op/s; 171/1770 objects degraded (9.661%) 2024-09-20T19:43:01.237 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 podman[73118]: 2024-09-20 19:43:01.062225583 +0000 UTC m=+0.906914647 container remove 685ab766bea088155784252dd2fd8668c013e70d2603d7540b268866867647ef (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-20T19:43:01.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 193369 10.14 MB/sec execute 351 sec latency 41.968 ms 2024-09-20T19:43:01.487 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 podman[73175]: 2024-09-20 19:43:01.189684539 +0000 UTC m=+0.018936614 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:43:01.488 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 podman[73175]: 2024-09-20 19:43:01.351697238 +0000 UTC m=+0.180949303 container create e0b40360570b5a200ce60ca67443ec3588ee8ae2d2b792f99636812aa1afbfdc (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-20T19:43:01.744 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 podman[73175]: 2024-09-20 19:43:01.722460019 +0000 UTC m=+0.551712093 container init e0b40360570b5a200ce60ca67443ec3588ee8ae2d2b792f99636812aa1afbfdc (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:43:01.745 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 podman[73175]: 2024-09-20 19:43:01.733218756 +0000 UTC m=+0.562470830 container start e0b40360570b5a200ce60ca67443ec3588ee8ae2d2b792f99636812aa1afbfdc (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-20T19:43:02.007 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:01 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 172/1773 objects degraded (9.701%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:02.008 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 ceph-osd[73193]: -- 172.21.15.116:0/1642868259 <== mon.0 v2:172.21.15.116:3300/0 3 ==== mon_map magic: 0 ==== 357+0+0 (secure 0 0 0) 0x561a9c16c1a0 con 0x561a9c132000 2024-09-20T19:43:02.008 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 ceph-osd[73193]: -- 172.21.15.116:0/1642868259 <== mon.0 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561a9c1381e0 con 0x561a9c132000 2024-09-20T19:43:02.008 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 bash[73175]: e0b40360570b5a200ce60ca67443ec3588ee8ae2d2b792f99636812aa1afbfdc 2024-09-20T19:43:02.008 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:01 smithi116 systemd[1]: Started Ceph osd.2 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:43:02.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:01 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 172/1773 objects degraded (9.701%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:02.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 193772 10.15 MB/sec execute 352 sec latency 49.572 ms 2024-09-20T19:43:03.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:03 smithi116 ceph-mon[63578]: pgmap v130: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.4 MiB/s wr, 391 op/s; 172/1773 objects degraded (9.701%) 2024-09-20T19:43:03.044 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:03 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:03.044 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:03 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:03.044 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:03 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:43:03.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 194215 10.14 MB/sec execute 353 sec latency 86.687 ms 2024-09-20T19:43:03.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:03 smithi118 ceph-mon[49197]: pgmap v130: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.4 MiB/s wr, 391 op/s; 172/1773 objects degraded (9.701%) 2024-09-20T19:43:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:03 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:03 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:03 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:43:04.075 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:03 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:43:03.887+0000 7effff978640 -1 Falling back to public interface 2024-09-20T19:43:04.302 INFO:teuthology.orchestra.run.smithi116.stdout: 5 194564 10.16 MB/sec execute 354 sec latency 61.093 ms 2024-09-20T19:43:05.114 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:05 smithi116 ceph-mon[63578]: pgmap v131: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 467 op/s; 165/1656 objects degraded (9.964%) 2024-09-20T19:43:05.303 INFO:teuthology.orchestra.run.smithi116.stdout: 5 194798 10.15 MB/sec execute 355 sec latency 74.744 ms 2024-09-20T19:43:05.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:05 smithi118 ceph-mon[49197]: pgmap v131: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 467 op/s; 165/1656 objects degraded (9.964%) 2024-09-20T19:43:06.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 195272 10.16 MB/sec execute 356 sec latency 33.929 ms 2024-09-20T19:43:07.154 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:07 smithi116 ceph-mon[63578]: pgmap v132: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 495 op/s; 166/1656 objects degraded (10.024%) 2024-09-20T19:43:07.303 INFO:teuthology.orchestra.run.smithi116.stdout: 5 195515 10.16 MB/sec execute 357 sec latency 53.693 ms 2024-09-20T19:43:07.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:07 smithi118 ceph-mon[49197]: pgmap v132: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 495 op/s; 166/1656 objects degraded (10.024%) 2024-09-20T19:43:08.178 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:08 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 166/1656 objects degraded (10.024%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:08.178 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:08 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:08.303 INFO:teuthology.orchestra.run.smithi116.stdout: 5 195672 10.15 MB/sec execute 358 sec latency 62.701 ms 2024-09-20T19:43:08.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:08 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 166/1656 objects degraded (10.024%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:08.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:08 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:09.207 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:09 smithi116 ceph-mon[63578]: pgmap v133: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 448 op/s; 166/1656 objects degraded (10.024%) 2024-09-20T19:43:09.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 195813 10.12 MB/sec execute 359 sec latency 81.260 ms 2024-09-20T19:43:09.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:09 smithi118 ceph-mon[49197]: pgmap v133: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 448 op/s; 166/1656 objects degraded (10.024%) 2024-09-20T19:43:10.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 196054 10.11 MB/sec execute 360 sec latency 88.176 ms 2024-09-20T19:43:10.760 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:10 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:43:10.699+0000 7effff978640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-20T19:43:11.280 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:11 smithi116 ceph-mon[63578]: pgmap v134: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 613 op/s; 172/1746 objects degraded (9.851%) 2024-09-20T19:43:11.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 196345 10.09 MB/sec execute 361 sec latency 87.847 ms 2024-09-20T19:43:11.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:11 smithi118 ceph-mon[49197]: pgmap v134: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 613 op/s; 172/1746 objects degraded (9.851%) 2024-09-20T19:43:11.531 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:11 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:43:11.471+0000 7effff978640 -1 osd.2 96 log_to_monitors true 2024-09-20T19:43:12.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 196580 10.07 MB/sec execute 362 sec latency 85.088 ms 2024-09-20T19:43:12.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:12 smithi116 ceph-mon[63578]: from='osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-20T19:43:12.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:12.310 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:12.311 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:43:12 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:43:12.157+0000 7efff7722640 -1 osd.2 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:43:12.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:12 smithi118 ceph-mon[49197]: from='osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-20T19:43:12.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:12.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:13.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 196782 10.06 MB/sec execute 363 sec latency 209.293 ms 2024-09-20T19:43:13.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: pgmap v135: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 425 op/s; 168/1737 objects degraded (9.672%) 2024-09-20T19:43:13.356 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: from='osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T19:43:13.357 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: osdmap e99: 8 total, 7 up, 8 in 2024-09-20T19:43:13.357 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: from='osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:43:13.357 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 168/1737 objects degraded (9.672%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:13.357 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:13.357 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:13.357 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: pgmap v135: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 425 op/s; 168/1737 objects degraded (9.672%) 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: from='osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: osdmap e99: 8 total, 7 up, 8 in 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: from='osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 168/1737 objects degraded (9.672%), 18 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:13.366 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:14.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 196803 10.03 MB/sec execute 364 sec latency 1209.319 ms 2024-09-20T19:43:14.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:14 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:43:14.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:14 smithi116 ceph-mon[63578]: osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004] boot 2024-09-20T19:43:14.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:14 smithi116 ceph-mon[63578]: osdmap e100: 8 total, 8 up, 8 in 2024-09-20T19:43:14.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:14 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T19:43:14.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:14 smithi116 ceph-mon[63578]: osdmap e101: 8 total, 8 up, 8 in 2024-09-20T19:43:14.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:14 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:43:14.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:14 smithi118 ceph-mon[49197]: osd.2 [v2:172.21.15.116:6818/1261032004,v1:172.21.15.116:6819/1261032004] boot 2024-09-20T19:43:14.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:14 smithi118 ceph-mon[49197]: osdmap e100: 8 total, 8 up, 8 in 2024-09-20T19:43:14.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:14 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T19:43:14.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:14 smithi118 ceph-mon[49197]: osdmap e101: 8 total, 8 up, 8 in 2024-09-20T19:43:15.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:15 smithi116 ceph-mon[63578]: pgmap v138: 97 pgs: 16 peering, 3 active+undersized, 6 active+undersized+degraded, 72 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 57/1569 objects degraded (3.633%) 2024-09-20T19:43:15.265 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:15 smithi116 ceph-mon[63578]: osdmap e102: 8 total, 8 up, 8 in 2024-09-20T19:43:15.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 196858 10.01 MB/sec execute 365 sec latency 2066.001 ms 2024-09-20T19:43:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:15 smithi118 ceph-mon[49197]: pgmap v138: 97 pgs: 16 peering, 3 active+undersized, 6 active+undersized+degraded, 72 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 57/1569 objects degraded (3.633%) 2024-09-20T19:43:15.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:15 smithi118 ceph-mon[49197]: osdmap e102: 8 total, 8 up, 8 in 2024-09-20T19:43:16.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 197252 10.03 MB/sec execute 366 sec latency 56.288 ms 2024-09-20T19:43:17.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 197579 10.01 MB/sec execute 367 sec latency 699.529 ms 2024-09-20T19:43:17.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:17 smithi116 ceph-mon[63578]: pgmap v141: 97 pgs: 1 activating+undersized+degraded+remapped, 16 peering, 1 active+recovery_wait+degraded, 1 active+undersized, 4 active+undersized+degraded, 74 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.5 MiB/s wr, 452 op/s; 69/1539 objects degraded (4.483%); 5.6 KiB/s, 0 objects/s recovering 2024-09-20T19:43:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:17 smithi116 ceph-mon[63578]: osdmap e103: 8 total, 8 up, 8 in 2024-09-20T19:43:17.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:17 smithi118 ceph-mon[49197]: pgmap v141: 97 pgs: 1 activating+undersized+degraded+remapped, 16 peering, 1 active+recovery_wait+degraded, 1 active+undersized, 4 active+undersized+degraded, 74 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.5 MiB/s wr, 452 op/s; 69/1539 objects degraded (4.483%); 5.6 KiB/s, 0 objects/s recovering 2024-09-20T19:43:17.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:17 smithi118 ceph-mon[49197]: osdmap e103: 8 total, 8 up, 8 in 2024-09-20T19:43:18.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 197868 10.00 MB/sec execute 368 sec latency 662.443 ms 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: osdmap e104: 8 total, 8 up, 8 in 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 69/1539 objects degraded (4.483%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: osdmap e105: 8 total, 8 up, 8 in 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:43:18.414 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: osdmap e104: 8 total, 8 up, 8 in 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 69/1539 objects degraded (4.483%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: osdmap e105: 8 total, 8 up, 8 in 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:43:18.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:19.176 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: pgmap v144: 97 pgs: 1 activating+undersized+degraded+remapped, 16 peering, 1 active+recovery_wait+degraded, 79 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 143 KiB/s wr, 21 op/s; 42/1539 objects degraded (2.729%); 35 KiB/s, 0 objects/s recovering 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-20T19:43:19.177 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:19 smithi116 ceph-mon[63578]: osdmap e106: 8 total, 8 up, 8 in 2024-09-20T19:43:19.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 198067 10.00 MB/sec execute 369 sec latency 868.210 ms 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: pgmap v144: 97 pgs: 1 activating+undersized+degraded+remapped, 16 peering, 1 active+recovery_wait+degraded, 79 active+clean; 328 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 143 KiB/s wr, 21 op/s; 42/1539 objects degraded (2.729%); 35 KiB/s, 0 objects/s recovering 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T19:43:19.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T19:43:19.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-20T19:43:19.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:19 smithi118 ceph-mon[49197]: osdmap e106: 8 total, 8 up, 8 in 2024-09-20T19:43:20.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 198337 10.00 MB/sec execute 370 sec latency 62.867 ms 2024-09-20T19:43:21.223 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:43:21.247 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:21 smithi116 ceph-mon[63578]: pgmap v147: 97 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+degraded, 94 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.0 MiB/s wr, 340 op/s; 42/1578 objects degraded (2.662%); 1.0 MiB/s, 36 objects/s recovering 2024-09-20T19:43:21.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 198606 9.99 MB/sec execute 371 sec latency 56.511 ms 2024-09-20T19:43:21.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:21 smithi118 ceph-mon[49197]: pgmap v147: 97 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+degraded, 94 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.0 MiB/s wr, 340 op/s; 42/1578 objects degraded (2.662%); 1.0 MiB/s, 36 objects/s recovering 2024-09-20T19:43:22.046 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:43:22.046 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (5m) 10s ago 14m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:43:22.046 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (2m) 10s ago 14m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:43:22.046 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (2m) 2m ago 14m 37.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:43:22.046 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (4m) 10s ago 14m 46.1M - dad864ee21e9 8a163a77164a 2024-09-20T19:43:22.046 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (8m) 10s ago 8m 37.2M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:43:22.046 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (8m) 2m ago 8m 14.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (5m) 10s ago 15m 496M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (4m) 2m ago 14m 448M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (3m) 10s ago 16m 51.0M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (3m) 2m ago 13m 38.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (8m) 10s ago 8m 285M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (8m) 2m ago 8m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (5m) 10s ago 14m 10.4M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (5m) 2m ago 13m 10.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (115s) 10s ago 13m 284M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (67s) 10s ago 12m 217M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (20s) 10s ago 12m 12.2M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (11m) 10s ago 11m 608M 1936M 17.2.0 e1d6a67b021e d61d06dce65a 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (11m) 2m ago 11m 517M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (11m) 2m ago 11m 318M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:43:22.047 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (10m) 2m ago 10m 512M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:43:22.048 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (10m) 2m ago 10m 383M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:43:22.048 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (4m) 10s ago 13m 51.0M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:43:22.285 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:22 smithi116 ceph-mon[63578]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:22.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 198907 9.99 MB/sec execute 372 sec latency 49.282 ms 2024-09-20T19:43:22.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:22 smithi118 ceph-mon[49197]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:22.690 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:43:22.690 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:43:22.690 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:43:22.690 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:22.690 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:43:22.690 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 3 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 7 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:43:22.691 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "9/23 daemons upgraded", 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:43:23.193 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:43:23.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 199236 9.99 MB/sec execute 373 sec latency 80.650 ms 2024-09-20T19:43:23.319 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:23 smithi116 ceph-mon[63578]: from='client.44235 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:23.319 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:23 smithi116 ceph-mon[63578]: pgmap v148: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.4 MiB/s wr, 257 op/s; 814 KiB/s, 33 objects/s recovering 2024-09-20T19:43:23.319 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:23 smithi116 ceph-mon[63578]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:23.319 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:23 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1578 objects degraded (2.662%), 2 pgs degraded) 2024-09-20T19:43:23.319 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:23 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1361496834' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:23.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:23 smithi118 ceph-mon[49197]: from='client.44235 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:23 smithi118 ceph-mon[49197]: pgmap v148: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.4 MiB/s wr, 257 op/s; 814 KiB/s, 33 objects/s recovering 2024-09-20T19:43:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:23 smithi118 ceph-mon[49197]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:23 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 42/1578 objects degraded (2.662%), 2 pgs degraded) 2024-09-20T19:43:23.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:23 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1361496834' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:23.802 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:43:23.803 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:43:24.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 199924 9.99 MB/sec execute 374 sec latency 56.348 ms 2024-09-20T19:43:24.328 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:24 smithi116 ceph-mon[63578]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:24.329 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:24 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/960160066' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:43:24.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:24 smithi118 ceph-mon[49197]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:24.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:24 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/960160066' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:43:25.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 200592 10.02 MB/sec execute 375 sec latency 24.867 ms 2024-09-20T19:43:25.336 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:25 smithi116 ceph-mon[63578]: pgmap v149: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.7 MiB/s wr, 516 op/s; 674 KiB/s, 28 objects/s recovering 2024-09-20T19:43:25.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:25 smithi118 ceph-mon[49197]: pgmap v149: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 5.7 MiB/s wr, 516 op/s; 674 KiB/s, 28 objects/s recovering 2024-09-20T19:43:26.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 200928 10.01 MB/sec execute 376 sec latency 95.081 ms 2024-09-20T19:43:27.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 201231 10.02 MB/sec execute 377 sec latency 59.565 ms 2024-09-20T19:43:27.383 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:27 smithi116 ceph-mon[63578]: pgmap v150: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.9 MiB/s wr, 441 op/s; 558 KiB/s, 23 objects/s recovering 2024-09-20T19:43:27.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:27 smithi118 ceph-mon[49197]: pgmap v150: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.9 MiB/s wr, 441 op/s; 558 KiB/s, 23 objects/s recovering 2024-09-20T19:43:28.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 201463 10.01 MB/sec execute 378 sec latency 95.298 ms 2024-09-20T19:43:28.605 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:28.606 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:28.606 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:28 smithi116 ceph-mon[63578]: pgmap v151: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 369 op/s; 464 KiB/s, 19 objects/s recovering 2024-09-20T19:43:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:28 smithi118 ceph-mon[49197]: pgmap v151: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.1 MiB/s wr, 369 op/s; 464 KiB/s, 19 objects/s recovering 2024-09-20T19:43:29.304 INFO:teuthology.orchestra.run.smithi116.stdout: 5 201775 10.01 MB/sec execute 379 sec latency 72.335 ms 2024-09-20T19:43:30.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 201956 9.99 MB/sec execute 380 sec latency 86.796 ms 2024-09-20T19:43:30.941 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:30 smithi116 ceph-mon[63578]: pgmap v152: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 33 KiB/s, 3 objects/s recovering 2024-09-20T19:43:31.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:30 smithi118 ceph-mon[49197]: pgmap v152: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 508 op/s; 33 KiB/s, 3 objects/s recovering 2024-09-20T19:43:31.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 202202 9.98 MB/sec execute 381 sec latency 72.727 ms 2024-09-20T19:43:32.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 202556 9.99 MB/sec execute 382 sec latency 68.638 ms 2024-09-20T19:43:32.982 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:32 smithi116 ceph-mon[63578]: pgmap v153: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 456 op/s; 30 KiB/s, 3 objects/s recovering 2024-09-20T19:43:32.982 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:32 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:33.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:32 smithi118 ceph-mon[49197]: pgmap v153: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 456 op/s; 30 KiB/s, 3 objects/s recovering 2024-09-20T19:43:33.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:32 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:33.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 202800 9.97 MB/sec execute 383 sec latency 94.036 ms 2024-09-20T19:43:34.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:33 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T19:43:34.044 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:33 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T19:43:34.044 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:33 smithi116 ceph-mon[63578]: Upgrade: osd.3 is safe to restart 2024-09-20T19:43:34.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:33 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T19:43:34.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:33 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T19:43:34.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:33 smithi118 ceph-mon[49197]: Upgrade: osd.3 is safe to restart 2024-09-20T19:43:34.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 203060 9.97 MB/sec execute 384 sec latency 69.841 ms 2024-09-20T19:43:35.058 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:34 smithi116 ceph-mon[63578]: pgmap v154: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 606 op/s 2024-09-20T19:43:35.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:34 smithi118 ceph-mon[49197]: pgmap v154: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 606 op/s 2024-09-20T19:43:35.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 203303 9.95 MB/sec execute 385 sec latency 72.124 ms 2024-09-20T19:43:36.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 203584 9.94 MB/sec execute 386 sec latency 48.345 ms 2024-09-20T19:43:36.862 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:36 smithi116 ceph-mon[63578]: Upgrade: Updating osd.3 2024-09-20T19:43:36.862 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:36.862 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T19:43:36.862 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:43:36.863 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:36 smithi116 ceph-mon[63578]: Deploying daemon osd.3 on smithi116 2024-09-20T19:43:36.863 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:36 smithi116 ceph-mon[63578]: pgmap v155: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-20T19:43:36.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:36 smithi118 ceph-mon[49197]: Upgrade: Updating osd.3 2024-09-20T19:43:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T19:43:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:43:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:36 smithi118 ceph-mon[49197]: Deploying daemon osd.3 on smithi116 2024-09-20T19:43:36.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:36 smithi118 ceph-mon[49197]: pgmap v155: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-20T19:43:37.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 203849 9.93 MB/sec execute 387 sec latency 61.841 ms 2024-09-20T19:43:38.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 204144 9.93 MB/sec execute 388 sec latency 71.538 ms 2024-09-20T19:43:38.907 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:38 smithi116 ceph-mon[63578]: pgmap v156: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-20T19:43:39.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:38 smithi118 ceph-mon[49197]: pgmap v156: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-20T19:43:39.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 204489 9.92 MB/sec execute 389 sec latency 46.922 ms 2024-09-20T19:43:40.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 204808 9.93 MB/sec execute 390 sec latency 62.313 ms 2024-09-20T19:43:40.735 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:40 smithi116 systemd[1]: Stopping Ceph osd.3 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:43:41.005 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:40 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[45107]: 2024-09-20T19:43:40.983+0000 7fbec1337700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:43:41.005 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:40 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[45107]: 2024-09-20T19:43:40.983+0000 7fbec1337700 -1 osd.3 106 *** Got signal Terminated *** 2024-09-20T19:43:41.005 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:40 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[45107]: 2024-09-20T19:43:40.983+0000 7fbec1337700 -1 osd.3 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:43:41.005 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:40 smithi116 ceph-mon[63578]: pgmap v157: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.6 MiB/s wr, 596 op/s 2024-09-20T19:43:41.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:40 smithi118 ceph-mon[49197]: pgmap v157: 97 pgs: 97 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.6 MiB/s wr, 596 op/s 2024-09-20T19:43:41.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 205012 9.92 MB/sec execute 391 sec latency 72.584 ms 2024-09-20T19:43:42.045 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:41 smithi116 ceph-mon[63578]: osd.3 marked itself down and dead 2024-09-20T19:43:42.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:41 smithi118 ceph-mon[49197]: osd.3 marked itself down and dead 2024-09-20T19:43:42.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 205252 9.91 MB/sec execute 392 sec latency 400.526 ms 2024-09-20T19:43:42.569 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:42 smithi116 podman[75071]: 2024-09-20 19:43:42.527183961 +0000 UTC m=+1.672430893 container died d61d06dce65aec4670050327963dcdb0df63070188d248848f975cd8b63b2e8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, distribution-scope=public, GIT_CLEAN=True, vcs-type=git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, name=centos-stream, RELEASE=HEAD, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: pgmap v158: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.3 MiB/s wr, 338 op/s 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: osdmap e107: 8 total, 7 up, 8 in 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 3]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 3]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 3]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2, 7, 3]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]} : dispatch 2024-09-20T19:43:42.829 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]} : dispatch 2024-09-20T19:43:42.830 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:42.830 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:42 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:43.116 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: pgmap v158: 97 pgs: 97 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.3 MiB/s wr, 338 op/s 2024-09-20T19:43:43.116 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:43:43.116 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: osdmap e107: 8 total, 7 up, 8 in 2024-09-20T19:43:43.116 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 3]} : dispatch 2024-09-20T19:43:43.116 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 3]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 3]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2, 7, 3]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]} : dispatch 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:43.117 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:42 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:43.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 205492 9.90 MB/sec execute 393 sec latency 965.601 ms 2024-09-20T19:43:43.363 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:43 smithi116 podman[75071]: 2024-09-20 19:43:43.306700975 +0000 UTC m=+2.451947896 container cleanup d61d06dce65aec4670050327963dcdb0df63070188d248848f975cd8b63b2e8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , name=centos-stream, architecture=x86_64, version=8, ceph=True, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.component=centos-stream-container) 2024-09-20T19:43:43.364 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:43 smithi116 bash[75071]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3 2024-09-20T19:43:43.676 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:43 smithi116 podman[75087]: 2024-09-20 19:43:43.600184034 +0000 UTC m=+1.068036286 container remove d61d06dce65aec4670050327963dcdb0df63070188d248848f975cd8b63b2e8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_CLEAN=True, architecture=x86_64, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, release=754, vcs-type=git, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-20T19:43:43.992 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 3]}]': finished 2024-09-20T19:43:43.992 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 3]}]': finished 2024-09-20T19:43:43.992 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-09-20T19:43:43.993 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 3]}]': finished 2024-09-20T19:43:43.993 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]}]': finished 2024-09-20T19:43:43.993 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-20T19:43:43.993 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2, 7, 3]}]': finished 2024-09-20T19:43:43.993 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2024-09-20T19:43:43.993 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-20T19:43:43.993 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:43 smithi116 ceph-mon[63578]: osdmap e108: 8 total, 7 up, 8 in 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 3]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.b", "id": [7, 3]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 3]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 2]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 2, 7, 3]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.f", "id": [4, 2]}]': finished 2024-09-20T19:43:44.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-20T19:43:44.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:43 smithi118 ceph-mon[49197]: osdmap e108: 8 total, 7 up, 8 in 2024-09-20T19:43:44.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 205574 9.88 MB/sec execute 394 sec latency 1473.778 ms 2024-09-20T19:43:44.373 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:43 smithi116 podman[75176]: 2024-09-20 19:43:43.894644746 +0000 UTC m=+0.015969138 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:43:44.373 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:44 smithi116 podman[75176]: 2024-09-20 19:43:44.222909114 +0000 UTC m=+0.344233495 container create 602e8d4a43cdbdbf4a4ec078aef987e51df10b3f44060ff9f055963be5bce871 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:43:44.826 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:44 smithi116 podman[75176]: 2024-09-20 19:43:44.638394108 +0000 UTC m=+0.759718500 container init 602e8d4a43cdbdbf4a4ec078aef987e51df10b3f44060ff9f055963be5bce871 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-20T19:43:44.827 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:44 smithi116 podman[75176]: 2024-09-20 19:43:44.647422819 +0000 UTC m=+0.768747199 container start 602e8d4a43cdbdbf4a4ec078aef987e51df10b3f44060ff9f055963be5bce871 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-20T19:43:44.827 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:44 smithi116 podman[75176]: 2024-09-20 19:43:44.825826745 +0000 UTC m=+0.947151136 container attach 602e8d4a43cdbdbf4a4ec078aef987e51df10b3f44060ff9f055963be5bce871 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-20T19:43:45.087 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:44 smithi116 podman[75176]: 2024-09-20 19:43:44.912697285 +0000 UTC m=+1.034021686 container died 602e8d4a43cdbdbf4a4ec078aef987e51df10b3f44060ff9f055963be5bce871 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-20T19:43:45.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:44 smithi116 ceph-mon[63578]: pgmap v161: 97 pgs: 9 active+undersized, 1 unknown, 11 peering, 3 stale+active+clean, 16 active+undersized+degraded, 57 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 474 op/s; 146/1599 objects degraded (9.131%) 2024-09-20T19:43:45.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:44 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 146/1599 objects degraded (9.131%), 16 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:45.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:44 smithi116 ceph-mon[63578]: osdmap e109: 8 total, 7 up, 8 in 2024-09-20T19:43:45.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:44 smithi118 ceph-mon[49197]: pgmap v161: 97 pgs: 9 active+undersized, 1 unknown, 11 peering, 3 stale+active+clean, 16 active+undersized+degraded, 57 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.1 MiB/s wr, 474 op/s; 146/1599 objects degraded (9.131%) 2024-09-20T19:43:45.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:44 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 146/1599 objects degraded (9.131%), 16 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:45.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:44 smithi118 ceph-mon[49197]: osdmap e109: 8 total, 7 up, 8 in 2024-09-20T19:43:45.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 205662 9.86 MB/sec execute 395 sec latency 2113.825 ms 2024-09-20T19:43:45.864 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:45 smithi116 ceph-mon[63578]: osdmap e110: 8 total, 7 up, 8 in 2024-09-20T19:43:45.864 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:45 smithi116 podman[75198]: 2024-09-20 19:43:45.734039365 +0000 UTC m=+0.805977550 container remove 602e8d4a43cdbdbf4a4ec078aef987e51df10b3f44060ff9f055963be5bce871 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-20T19:43:45.864 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:45 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-20T19:43:45.864 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:45 smithi116 systemd[1]: Stopped Ceph osd.3 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:43:45.864 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:45 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.3.service: Consumed 3min 58.076s CPU time. 2024-09-20T19:43:46.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:45 smithi118 ceph-mon[49197]: osdmap e110: 8 total, 7 up, 8 in 2024-09-20T19:43:46.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 205895 9.86 MB/sec execute 396 sec latency 46.012 ms 2024-09-20T19:43:46.377 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 systemd[1]: Starting Ceph osd.3 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:43:46.378 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 podman[75296]: 2024-09-20 19:43:46.262598826 +0000 UTC m=+0.017943362 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:43:46.630 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 podman[75296]: 2024-09-20 19:43:46.392557159 +0000 UTC m=+0.147901668 container create 45bfbeeb1b06c8c87c62e3883c5153ada9d3fb50b9d6ec0204bae52a2ec77603 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-20T19:43:46.889 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:46 smithi116 ceph-mon[63578]: pgmap v164: 97 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized, 1 unknown, 11 peering, 22 active+undersized+degraded, 49 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 464 op/s; 186/1602 objects degraded (11.610%); 535/1602 objects misplaced (33.396%) 2024-09-20T19:43:46.890 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:46 smithi116 ceph-mon[63578]: osdmap e111: 8 total, 7 up, 8 in 2024-09-20T19:43:46.890 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 podman[75296]: 2024-09-20 19:43:46.750245193 +0000 UTC m=+0.505589712 container init 45bfbeeb1b06c8c87c62e3883c5153ada9d3fb50b9d6ec0204bae52a2ec77603 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:43:46.890 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 podman[75296]: 2024-09-20 19:43:46.762438922 +0000 UTC m=+0.517783437 container start 45bfbeeb1b06c8c87c62e3883c5153ada9d3fb50b9d6ec0204bae52a2ec77603 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-20T19:43:46.890 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 podman[75296]: 2024-09-20 19:43:46.850295511 +0000 UTC m=+0.605640022 container attach 45bfbeeb1b06c8c87c62e3883c5153ada9d3fb50b9d6ec0204bae52a2ec77603 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:43:47.164 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:43:47.165 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 bash[75296]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:43:47.165 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:43:47.165 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:46 smithi116 bash[75296]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:43:47.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:46 smithi118 ceph-mon[49197]: pgmap v164: 97 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized, 1 unknown, 11 peering, 22 active+undersized+degraded, 49 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 MiB/s wr, 464 op/s; 186/1602 objects degraded (11.610%); 535/1602 objects misplaced (33.396%) 2024-09-20T19:43:47.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:46 smithi118 ceph-mon[49197]: osdmap e111: 8 total, 7 up, 8 in 2024-09-20T19:43:47.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 206076 9.84 MB/sec execute 397 sec latency 77.316 ms 2024-09-20T19:43:47.677 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T19:43:47.677 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 bash[75296]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T19:43:47.678 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T19:43:47.678 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 bash[75296]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 bash[75296]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 bash[75296]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 bash[75296]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 bash[75296]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate[75312]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-20T19:43:47.929 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 bash[75296]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-20T19:43:47.929 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:47 smithi116 ceph-mon[63578]: osdmap e112: 8 total, 7 up, 8 in 2024-09-20T19:43:48.182 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:47 smithi116 podman[75296]: 2024-09-20 19:43:47.936719902 +0000 UTC m=+1.692064426 container died 45bfbeeb1b06c8c87c62e3883c5153ada9d3fb50b9d6ec0204bae52a2ec77603 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS) 2024-09-20T19:43:48.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:47 smithi118 ceph-mon[49197]: osdmap e112: 8 total, 7 up, 8 in 2024-09-20T19:43:48.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 206387 9.85 MB/sec execute 398 sec latency 52.424 ms 2024-09-20T19:43:48.960 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:48 smithi116 ceph-mon[63578]: pgmap v167: 97 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized, 1 unknown, 6 peering, 25 active+undersized+degraded, 49 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 743 KiB/s wr, 32 op/s; 200/1602 objects degraded (12.484%); 535/1602 objects misplaced (33.396%) 2024-09-20T19:43:48.960 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:48 smithi116 ceph-mon[63578]: osdmap e113: 8 total, 7 up, 8 in 2024-09-20T19:43:49.134 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:49 smithi116 podman[75459]: 2024-09-20 19:43:49.068592427 +0000 UTC m=+1.125323203 container remove 45bfbeeb1b06c8c87c62e3883c5153ada9d3fb50b9d6ec0204bae52a2ec77603 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-activate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:43:49.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:48 smithi118 ceph-mon[49197]: pgmap v167: 97 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 14 active+undersized, 1 unknown, 6 peering, 25 active+undersized+degraded, 49 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 743 KiB/s wr, 32 op/s; 200/1602 objects degraded (12.484%); 535/1602 objects misplaced (33.396%) 2024-09-20T19:43:49.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:48 smithi118 ceph-mon[49197]: osdmap e113: 8 total, 7 up, 8 in 2024-09-20T19:43:49.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 206776 9.83 MB/sec execute 399 sec latency 74.776 ms 2024-09-20T19:43:49.470 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:49 smithi116 podman[75516]: 2024-09-20 19:43:49.222218059 +0000 UTC m=+0.024143224 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:43:49.470 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:49 smithi116 podman[75516]: 2024-09-20 19:43:49.361519517 +0000 UTC m=+0.163444672 container create e3b204ab779476c968568b7990d08d61d7ca290754c39fc5ef3f6a3381a8c4ac (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD) 2024-09-20T19:43:49.985 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:49 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 204/1605 objects degraded (12.710%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:50.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:49 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 204/1605 objects degraded (12.710%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:50.242 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:50 smithi116 podman[75516]: 2024-09-20 19:43:50.115384361 +0000 UTC m=+0.917309513 container init e3b204ab779476c968568b7990d08d61d7ca290754c39fc5ef3f6a3381a8c4ac (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-20T19:43:50.242 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:50 smithi116 podman[75516]: 2024-09-20 19:43:50.125304562 +0000 UTC m=+0.927229718 container start e3b204ab779476c968568b7990d08d61d7ca290754c39fc5ef3f6a3381a8c4ac (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-20T19:43:50.243 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:50 smithi116 ceph-osd[75536]: -- 172.21.15.116:0/4129343339 <== mon.0 v2:172.21.15.116:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bea2a161e0 con 0x55bea1bb9c00 2024-09-20T19:43:50.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 207061 9.83 MB/sec execute 400 sec latency 41.527 ms 2024-09-20T19:43:50.496 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:50 smithi116 bash[75516]: e3b204ab779476c968568b7990d08d61d7ca290754c39fc5ef3f6a3381a8c4ac 2024-09-20T19:43:50.496 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:50 smithi116 systemd[1]: Started Ceph osd.3 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:43:51.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 207343 9.83 MB/sec execute 401 sec latency 52.235 ms 2024-09-20T19:43:51.536 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:51 smithi116 ceph-mon[63578]: pgmap v169: 97 pgs: 7 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s wr, 234 op/s; 204/1605 objects degraded (12.710%); 588/1605 objects misplaced (36.636%); 1.6 MiB/s, 8 keys/s, 120 objects/s recovering 2024-09-20T19:43:51.537 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:51.537 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:51.537 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:51 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:43:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:51 smithi118 ceph-mon[49197]: pgmap v169: 97 pgs: 7 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s wr, 234 op/s; 204/1605 objects degraded (12.710%); 588/1605 objects misplaced (36.636%); 1.6 MiB/s, 8 keys/s, 120 objects/s recovering 2024-09-20T19:43:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:51 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:51 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:51 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:43:52.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 207767 9.83 MB/sec execute 402 sec latency 50.007 ms 2024-09-20T19:43:52.314 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:43:52 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[75532]: 2024-09-20T19:43:52.266+0000 7fb13312a640 -1 Falling back to public interface 2024-09-20T19:43:53.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 208232 9.83 MB/sec execute 403 sec latency 56.085 ms 2024-09-20T19:43:53.580 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:53 smithi116 ceph-mon[63578]: pgmap v170: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.2 MiB/s wr, 347 op/s; 214/1632 objects degraded (13.113%); 585/1632 objects misplaced (35.846%); 1.3 MiB/s, 6 keys/s, 98 objects/s recovering 2024-09-20T19:43:53.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:53 smithi118 ceph-mon[49197]: pgmap v170: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.2 MiB/s wr, 347 op/s; 214/1632 objects degraded (13.113%); 585/1632 objects misplaced (35.846%); 1.3 MiB/s, 6 keys/s, 98 objects/s recovering 2024-09-20T19:43:54.305 INFO:teuthology.orchestra.run.smithi116.stdout: 5 208469 9.83 MB/sec execute 404 sec latency 38.890 ms 2024-09-20T19:43:54.434 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:43:55.297 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:43:55.297 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (6m) 43s ago 15m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:43:55.297 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (3m) 43s ago 15m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:43:55.297 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (3m) 2m ago 14m 37.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:43:55.297 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (5m) 43s ago 15m 46.1M - dad864ee21e9 8a163a77164a 2024-09-20T19:43:55.297 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (9m) 43s ago 9m 37.2M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:43:55.297 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (9m) 2m ago 9m 14.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (5m) 43s ago 16m 496M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (4m) 2m ago 14m 448M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (4m) 43s ago 16m 51.0M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (3m) 2m ago 14m 38.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (8m) 43s ago 8m 285M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (8m) 2m ago 8m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (6m) 43s ago 14m 10.4M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (5m) 2m ago 14m 10.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (2m) 43s ago 13m 284M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (100s) 43s ago 13m 217M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (54s) 43s ago 12m 12.2M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 starting - - - 1936M 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (12m) 2m ago 12m 517M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:43:55.298 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (11m) 2m ago 11m 318M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:43:55.299 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (11m) 2m ago 11m 512M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:43:55.299 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (10m) 2m ago 10m 383M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:43:55.299 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (4m) 43s ago 14m 51.0M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:43:55.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 208652 9.82 MB/sec execute 405 sec latency 61.114 ms 2024-09-20T19:43:55.656 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:55 smithi116 ceph-mon[63578]: pgmap v171: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 224/1707 objects degraded (13.122%); 577/1707 objects misplaced (33.802%); 1.0 MiB/s, 5 keys/s, 89 objects/s recovering 2024-09-20T19:43:55.666 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:55 smithi116 ceph-mon[63578]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:55.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:55 smithi118 ceph-mon[49197]: pgmap v171: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 224/1707 objects degraded (13.122%); 577/1707 objects misplaced (33.802%); 1.0 MiB/s, 5 keys/s, 89 objects/s recovering 2024-09-20T19:43:55.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:55 smithi118 ceph-mon[49197]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-20T19:43:55.871 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 3 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 7 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:43:55.872 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:43:56.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 208885 9.81 MB/sec execute 406 sec latency 52.058 ms 2024-09-20T19:43:56.309 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:43:56.309 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:43:56.309 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "9/23 daemons upgraded", 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:43:56.310 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:43:56.442 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:56 smithi116 ceph-mon[63578]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:56.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:56 smithi116 ceph-mon[63578]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:56.443 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:56 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2948692699' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:56.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:56 smithi118 ceph-mon[49197]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:56.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:56 smithi118 ceph-mon[49197]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:56.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:56 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2948692699' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:43:56.959 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 osds down; Degraded data redundancy: 216/1692 objects degraded (12.766%), 27 pgs degraded; 5 mgr modules have recently crashed 2024-09-20T19:43:56.959 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:43:56.959 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:43:56.959 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T19:43:56.959 INFO:teuthology.orchestra.run.smithi116.stdout: osd.3 (root=default,host=smithi116) is down 2024-09-20T19:43:56.959 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 216/1692 objects degraded (12.766%), 27 pgs degraded 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.e is active+undersized+degraded, acting [2,4] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-20T19:43:56.960 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.e is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.f is active+recovering+undersized+degraded+remapped, acting [7,0] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-20T19:43:56.961 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:43:56.962 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:43:56.962 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:43:56.962 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:43:56.962 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:43:56.962 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:43:57.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 209121 9.80 MB/sec execute 407 sec latency 76.293 ms 2024-09-20T19:43:57.465 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:57 smithi116 ceph-mon[63578]: pgmap v172: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 216/1692 objects degraded (12.766%); 578/1692 objects misplaced (34.161%); 925 KiB/s, 4 keys/s, 79 objects/s recovering 2024-09-20T19:43:57.466 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:57 smithi116 ceph-mon[63578]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:57.466 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:57 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 216/1692 objects degraded (12.766%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:57.466 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:57 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1969504171' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:43:57.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:57 smithi118 ceph-mon[49197]: pgmap v172: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 453 op/s; 216/1692 objects degraded (12.766%); 578/1692 objects misplaced (34.161%); 925 KiB/s, 4 keys/s, 79 objects/s recovering 2024-09-20T19:43:57.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:57 smithi118 ceph-mon[49197]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:43:57.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:57 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 216/1692 objects degraded (12.766%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T19:43:57.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:57 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1969504171' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:43:58.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 209347 9.79 MB/sec execute 408 sec latency 60.897 ms 2024-09-20T19:43:58.751 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:58.752 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:58.752 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:58 smithi116 ceph-mon[63578]: pgmap v173: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 402 op/s; 216/1689 objects degraded (12.789%); 578/1689 objects misplaced (34.221%); 820 KiB/s, 3 keys/s, 70 objects/s recovering 2024-09-20T19:43:58.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:43:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:43:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:58 smithi118 ceph-mon[49197]: pgmap v173: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 402 op/s; 216/1689 objects degraded (12.789%); 578/1689 objects misplaced (34.221%); 820 KiB/s, 3 keys/s, 70 objects/s recovering 2024-09-20T19:43:59.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 209560 9.78 MB/sec execute 409 sec latency 60.086 ms 2024-09-20T19:44:00.076 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:00.076 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:00.077 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:00.077 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:43:59 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:00.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 209827 9.78 MB/sec execute 410 sec latency 57.108 ms 2024-09-20T19:44:00.331 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:44:00 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[75532]: 2024-09-20T19:44:00.297+0000 7fb13312a640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-20T19:44:00.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:00.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:00.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:00.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:43:59 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:01.115 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:00 smithi116 ceph-mon[63578]: pgmap v174: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.7 MiB/s wr, 471 op/s; 203/1572 objects degraded (12.913%); 574/1572 objects misplaced (36.514%); 691 KiB/s, 3 keys/s, 69 objects/s recovering 2024-09-20T19:44:01.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 210035 9.77 MB/sec execute 411 sec latency 62.764 ms 2024-09-20T19:44:01.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:00 smithi118 ceph-mon[49197]: pgmap v174: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.7 MiB/s wr, 471 op/s; 203/1572 objects degraded (12.913%); 574/1572 objects misplaced (36.514%); 691 KiB/s, 3 keys/s, 69 objects/s recovering 2024-09-20T19:44:01.617 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:44:01 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[75532]: 2024-09-20T19:44:01.488+0000 7fb13312a640 -1 osd.3 106 log_to_monitors true 2024-09-20T19:44:02.131 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:01 smithi116 ceph-mon[63578]: from='osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-20T19:44:02.132 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:44:02 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[75532]: 2024-09-20T19:44:02.046+0000 7fb12aed4640 -1 osd.3 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:44:02.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 210302 9.76 MB/sec execute 412 sec latency 77.260 ms 2024-09-20T19:44:02.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:01 smithi118 ceph-mon[49197]: from='osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-20T19:44:03.143 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:03 smithi116 ceph-mon[63578]: pgmap v175: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 393 op/s; 207/1590 objects degraded (13.019%); 574/1590 objects misplaced (36.101%); 0 B/s, 19 objects/s recovering 2024-09-20T19:44:03.144 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:03 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 207/1590 objects degraded (13.019%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:03.144 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:03 smithi116 ceph-mon[63578]: from='osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T19:44:03.144 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:03 smithi116 ceph-mon[63578]: osdmap e114: 8 total, 7 up, 8 in 2024-09-20T19:44:03.144 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:03 smithi116 ceph-mon[63578]: from='osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:44:03.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 210595 9.75 MB/sec execute 413 sec latency 261.208 ms 2024-09-20T19:44:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:03 smithi118 ceph-mon[49197]: pgmap v175: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 25 active+undersized+degraded, 49 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 393 op/s; 207/1590 objects degraded (13.019%); 574/1590 objects misplaced (36.101%); 0 B/s, 19 objects/s recovering 2024-09-20T19:44:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:03 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 207/1590 objects degraded (13.019%), 27 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:03 smithi118 ceph-mon[49197]: from='osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T19:44:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:03 smithi118 ceph-mon[49197]: osdmap e114: 8 total, 7 up, 8 in 2024-09-20T19:44:03.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:03 smithi118 ceph-mon[49197]: from='osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi116", "root=default"]} : dispatch 2024-09-20T19:44:04.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 210625 9.72 MB/sec execute 414 sec latency 1261.234 ms 2024-09-20T19:44:04.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:04 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:44:04.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:04 smithi116 ceph-mon[63578]: osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161] boot 2024-09-20T19:44:04.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:04 smithi116 ceph-mon[63578]: osdmap e115: 8 total, 8 up, 8 in 2024-09-20T19:44:04.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:04 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T19:44:04.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:04 smithi116 ceph-mon[63578]: osdmap e116: 8 total, 8 up, 8 in 2024-09-20T19:44:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:04 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:44:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:04 smithi118 ceph-mon[49197]: osd.3 [v2:172.21.15.116:6826/62973161,v1:172.21.15.116:6827/62973161] boot 2024-09-20T19:44:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:04 smithi118 ceph-mon[49197]: osdmap e115: 8 total, 8 up, 8 in 2024-09-20T19:44:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:04 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T19:44:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:04 smithi118 ceph-mon[49197]: osdmap e116: 8 total, 8 up, 8 in 2024-09-20T19:44:05.113 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:05 smithi116 ceph-mon[63578]: pgmap v178: 97 pgs: 14 peering, 6 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized, 18 active+undersized+degraded, 49 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 420 op/s; 136/1653 objects degraded (8.227%); 533/1653 objects misplaced (32.244%); 0 B/s, 28 objects/s recovering 2024-09-20T19:44:05.113 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:05 smithi116 ceph-mon[63578]: 2.b starting backfill to osd.3 from (0'0,0'0] MAX to 98'8622 2024-09-20T19:44:05.114 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:05 smithi116 ceph-mon[63578]: osdmap e117: 8 total, 8 up, 8 in 2024-09-20T19:44:05.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 210648 9.70 MB/sec execute 415 sec latency 2195.664 ms 2024-09-20T19:44:05.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:05 smithi118 ceph-mon[49197]: pgmap v178: 97 pgs: 14 peering, 6 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized, 18 active+undersized+degraded, 49 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 420 op/s; 136/1653 objects degraded (8.227%); 533/1653 objects misplaced (32.244%); 0 B/s, 28 objects/s recovering 2024-09-20T19:44:05.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:05 smithi118 ceph-mon[49197]: 2.b starting backfill to osd.3 from (0'0,0'0] MAX to 98'8622 2024-09-20T19:44:05.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:05 smithi118 ceph-mon[49197]: osdmap e117: 8 total, 8 up, 8 in 2024-09-20T19:44:06.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 210948 9.71 MB/sec execute 416 sec latency 286.768 ms 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: pgmap v181: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 14 peering, 6 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 14 active+undersized+degraded, 54 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.2 MiB/s wr, 424 op/s; 1682/1662 objects degraded (101.203%); 1.1 MiB/s, 33 objects/s recovering 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: osdmap e118: 8 total, 8 up, 8 in 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.189 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:44:07.190 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:07.190 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:07.190 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:07.190 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T19:44:07.190 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T19:44:07.190 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:07 smithi116 ceph-mon[63578]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-20T19:44:07.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 211158 9.70 MB/sec execute 417 sec latency 1019.402 ms 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: pgmap v181: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 14 peering, 6 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 14 active+undersized+degraded, 54 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.2 MiB/s wr, 424 op/s; 1682/1662 objects degraded (101.203%); 1.1 MiB/s, 33 objects/s recovering 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: osdmap e118: 8 total, 8 up, 8 in 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:07.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:07.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:07.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T19:44:07.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T19:44:07.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:07 smithi118 ceph-mon[49197]: Upgrade: unsafe to stop osd(s) at this time (15 PGs are or would become offline) 2024-09-20T19:44:08.223 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:08 smithi116 ceph-mon[63578]: osdmap e119: 8 total, 8 up, 8 in 2024-09-20T19:44:08.224 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:08 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 1682/1662 objects degraded (101.203%), 17 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:08.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 211428 9.69 MB/sec execute 418 sec latency 942.248 ms 2024-09-20T19:44:08.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:08 smithi118 ceph-mon[49197]: osdmap e119: 8 total, 8 up, 8 in 2024-09-20T19:44:08.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:08 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 1682/1662 objects degraded (101.203%), 17 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:09.260 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:09 smithi116 ceph-mon[63578]: pgmap v184: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 17 peering, 4 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 69 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 692 KiB/s wr, 52 op/s; 1629/1650 objects degraded (98.727%); 2.3 MiB/s, 6 keys/s, 23 objects/s recovering 2024-09-20T19:44:09.261 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:09 smithi116 ceph-mon[63578]: osdmap e120: 8 total, 8 up, 8 in 2024-09-20T19:44:09.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 211796 9.69 MB/sec execute 419 sec latency 64.920 ms 2024-09-20T19:44:09.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:09 smithi118 ceph-mon[49197]: pgmap v184: 97 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 17 peering, 4 remapped+peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 69 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 692 KiB/s wr, 52 op/s; 1629/1650 objects degraded (98.727%); 2.3 MiB/s, 6 keys/s, 23 objects/s recovering 2024-09-20T19:44:09.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:09 smithi118 ceph-mon[49197]: osdmap e120: 8 total, 8 up, 8 in 2024-09-20T19:44:10.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 212078 9.69 MB/sec execute 420 sec latency 167.920 ms 2024-09-20T19:44:11.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 212367 9.68 MB/sec execute 421 sec latency 1007.513 ms 2024-09-20T19:44:11.337 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:11 smithi116 ceph-mon[63578]: pgmap v186: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 3 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 294 op/s; 1675/1617 objects degraded (103.587%); 2/1617 objects misplaced (0.124%); 5.4 MiB/s, 5 keys/s, 107 objects/s recovering 2024-09-20T19:44:11.338 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:11 smithi116 ceph-mon[63578]: osdmap e121: 8 total, 8 up, 8 in 2024-09-20T19:44:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:11 smithi118 ceph-mon[49197]: pgmap v186: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 3 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 294 op/s; 1675/1617 objects degraded (103.587%); 2/1617 objects misplaced (0.124%); 5.4 MiB/s, 5 keys/s, 107 objects/s recovering 2024-09-20T19:44:11.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:11 smithi118 ceph-mon[49197]: osdmap e121: 8 total, 8 up, 8 in 2024-09-20T19:44:12.130 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:12 smithi116 ceph-mon[63578]: osdmap e122: 8 total, 8 up, 8 in 2024-09-20T19:44:12.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 212725 9.68 MB/sec execute 422 sec latency 684.502 ms 2024-09-20T19:44:12.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:12 smithi118 ceph-mon[49197]: osdmap e122: 8 total, 8 up, 8 in 2024-09-20T19:44:13.306 INFO:teuthology.orchestra.run.smithi116.stdout: 5 212895 9.67 MB/sec execute 423 sec latency 911.694 ms 2024-09-20T19:44:13.361 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:13 smithi116 ceph-mon[63578]: pgmap v189: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 3 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 1669/1608 objects degraded (103.794%); 2/1608 objects misplaced (0.124%); 4.6 MiB/s, 5 keys/s, 105 objects/s recovering 2024-09-20T19:44:13.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:13 smithi116 ceph-mon[63578]: osdmap e123: 8 total, 8 up, 8 in 2024-09-20T19:44:13.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:13 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 1669/1608 objects degraded (103.794%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:13.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:13.362 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:13.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:13 smithi118 ceph-mon[49197]: pgmap v189: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 3 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 85 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.1 MiB/s wr, 394 op/s; 1669/1608 objects degraded (103.794%); 2/1608 objects misplaced (0.124%); 4.6 MiB/s, 5 keys/s, 105 objects/s recovering 2024-09-20T19:44:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:13 smithi118 ceph-mon[49197]: osdmap e123: 8 total, 8 up, 8 in 2024-09-20T19:44:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:13 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 1669/1608 objects degraded (103.794%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:13.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:14.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 213152 9.67 MB/sec execute 424 sec latency 919.831 ms 2024-09-20T19:44:14.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:14 smithi116 ceph-mon[63578]: osdmap e124: 8 total, 8 up, 8 in 2024-09-20T19:44:14.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:14 smithi118 ceph-mon[49197]: osdmap e124: 8 total, 8 up, 8 in 2024-09-20T19:44:15.150 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:15 smithi116 ceph-mon[63578]: pgmap v192: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 93 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 331 op/s; 1601/1635 objects degraded (97.920%); 1.5 MiB/s, 98 objects/s recovering 2024-09-20T19:44:15.151 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:15 smithi116 ceph-mon[63578]: osdmap e125: 8 total, 8 up, 8 in 2024-09-20T19:44:15.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 213634 9.68 MB/sec execute 425 sec latency 54.359 ms 2024-09-20T19:44:15.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:15 smithi118 ceph-mon[49197]: pgmap v192: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 93 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 331 op/s; 1601/1635 objects degraded (97.920%); 1.5 MiB/s, 98 objects/s recovering 2024-09-20T19:44:15.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:15 smithi118 ceph-mon[49197]: osdmap e125: 8 total, 8 up, 8 in 2024-09-20T19:44:16.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 213963 9.66 MB/sec execute 426 sec latency 893.782 ms 2024-09-20T19:44:16.413 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:16 smithi116 ceph-mon[63578]: osdmap e126: 8 total, 8 up, 8 in 2024-09-20T19:44:16.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:16 smithi118 ceph-mon[49197]: osdmap e126: 8 total, 8 up, 8 in 2024-09-20T19:44:17.214 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:17 smithi116 ceph-mon[63578]: pgmap v195: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 93 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 392 op/s; 562/1626 objects degraded (34.563%); 1.4 MiB/s, 229 objects/s recovering 2024-09-20T19:44:17.214 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:17 smithi116 ceph-mon[63578]: osdmap e127: 8 total, 8 up, 8 in 2024-09-20T19:44:17.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 214161 9.65 MB/sec execute 427 sec latency 92.612 ms 2024-09-20T19:44:17.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:17 smithi118 ceph-mon[49197]: pgmap v195: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 93 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 392 op/s; 562/1626 objects degraded (34.563%); 1.4 MiB/s, 229 objects/s recovering 2024-09-20T19:44:17.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:17 smithi118 ceph-mon[49197]: osdmap e127: 8 total, 8 up, 8 in 2024-09-20T19:44:18.250 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:18 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 562/1626 objects degraded (34.563%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:18.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 214292 9.64 MB/sec execute 428 sec latency 58.446 ms 2024-09-20T19:44:18.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:18 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 562/1626 objects degraded (34.563%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:19.293 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:19 smithi116 ceph-mon[63578]: pgmap v197: 97 pgs: 1 active+recovery_wait+degraded, 96 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 591 op/s; 13/1629 objects degraded (0.798%); 256 KiB/s, 309 objects/s recovering 2024-09-20T19:44:19.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 214575 9.65 MB/sec execute 429 sec latency 62.612 ms 2024-09-20T19:44:19.464 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:19 smithi118 ceph-mon[49197]: pgmap v197: 97 pgs: 1 active+recovery_wait+degraded, 96 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 591 op/s; 13/1629 objects degraded (0.798%); 256 KiB/s, 309 objects/s recovering 2024-09-20T19:44:20.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 214900 9.64 MB/sec execute 430 sec latency 51.876 ms 2024-09-20T19:44:20.313 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:20 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1629 objects degraded (0.798%), 1 pg degraded) 2024-09-20T19:44:20.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:20 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1629 objects degraded (0.798%), 1 pg degraded) 2024-09-20T19:44:21.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 215260 9.64 MB/sec execute 431 sec latency 66.140 ms 2024-09-20T19:44:21.348 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:21 smithi116 ceph-mon[63578]: pgmap v198: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 496 op/s; 79 KiB/s, 180 objects/s recovering 2024-09-20T19:44:21.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:21 smithi118 ceph-mon[49197]: pgmap v198: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 496 op/s; 79 KiB/s, 180 objects/s recovering 2024-09-20T19:44:22.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 215499 9.63 MB/sec execute 432 sec latency 49.039 ms 2024-09-20T19:44:22.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T19:44:22.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:22 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T19:44:22.389 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:22 smithi116 ceph-mon[63578]: Upgrade: osd.4 is safe to restart 2024-09-20T19:44:22.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T19:44:22.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:22 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T19:44:22.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:22 smithi118 ceph-mon[49197]: Upgrade: osd.4 is safe to restart 2024-09-20T19:44:23.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 215663 9.61 MB/sec execute 433 sec latency 67.101 ms 2024-09-20T19:44:23.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:23 smithi116 ceph-mon[63578]: pgmap v199: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 399 op/s; 62 KiB/s, 142 objects/s recovering 2024-09-20T19:44:23.416 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:23.416 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T19:44:23.416 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:44:23.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:23 smithi118 ceph-mon[49197]: pgmap v199: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 399 op/s; 62 KiB/s, 142 objects/s recovering 2024-09-20T19:44:23.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:23.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T19:44:23.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:44:24.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 215932 9.61 MB/sec execute 434 sec latency 60.529 ms 2024-09-20T19:44:24.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:24 smithi116 ceph-mon[63578]: Upgrade: Updating osd.4 2024-09-20T19:44:24.431 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:24 smithi116 ceph-mon[63578]: Deploying daemon osd.4 on smithi118 2024-09-20T19:44:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:24 smithi118 ceph-mon[49197]: Upgrade: Updating osd.4 2024-09-20T19:44:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:24 smithi118 ceph-mon[49197]: Deploying daemon osd.4 on smithi118 2024-09-20T19:44:25.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 216226 9.60 MB/sec execute 435 sec latency 66.059 ms 2024-09-20T19:44:25.501 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:25 smithi116 ceph-mon[63578]: pgmap v200: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.7 MiB/s wr, 488 op/s; 54 KiB/s, 65 objects/s recovering 2024-09-20T19:44:25.546 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:25 smithi118 ceph-mon[49197]: pgmap v200: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.7 MiB/s wr, 488 op/s; 54 KiB/s, 65 objects/s recovering 2024-09-20T19:44:26.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 216423 9.59 MB/sec execute 436 sec latency 67.562 ms 2024-09-20T19:44:27.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:26 smithi118 systemd[1]: Stopping Ceph osd.4 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:44:27.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 216641 9.59 MB/sec execute 437 sec latency 79.114 ms 2024-09-20T19:44:27.369 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:27 smithi116 ceph-mon[63578]: pgmap v201: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 427 op/s; 47 KiB/s, 56 objects/s recovering 2024-09-20T19:44:27.370 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:27 smithi116 ceph-mon[63578]: osd.4 marked itself down and dead 2024-09-20T19:44:27.423 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:44:27.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:27 smithi118 ceph-mon[49197]: pgmap v201: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 427 op/s; 47 KiB/s, 56 objects/s recovering 2024-09-20T19:44:27.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:27 smithi118 ceph-mon[49197]: osd.4 marked itself down and dead 2024-09-20T19:44:27.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[29917]: 2024-09-20T19:44:27.263+0000 7fbf446fd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:44:27.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[29917]: 2024-09-20T19:44:27.263+0000 7fbf446fd700 -1 osd.4 127 *** Got signal Terminated *** 2024-09-20T19:44:27.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[29917]: 2024-09-20T19:44:27.263+0000 7fbf446fd700 -1 osd.4 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:44:28.079 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:44:28.079 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (6m) 29s ago 15m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:44:28.079 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (3m) 29s ago 15m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:44:28.079 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (3m) 3m ago 15m 37.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (5m) 29s ago 15m 47.1M - dad864ee21e9 8a163a77164a 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (9m) 29s ago 9m 37.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (9m) 3m ago 9m 14.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (6m) 29s ago 17m 497M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (5m) 3m ago 15m 448M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (4m) 29s ago 17m 51.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (4m) 3m ago 15m 38.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (9m) 29s ago 9m 290M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (9m) 3m ago 9m 85.6M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (6m) 29s ago 15m 10.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 3m ago 14m 10.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (3m) 29s ago 14m 343M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (2m) 29s ago 13m 255M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (86s) 29s ago 13m 214M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:44:28.080 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (38s) 29s ago 12m 14.3M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:44:28.081 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (12m) 3m ago 12m 517M 2704M 17.2.0 e1d6a67b021e 53207408a89d 2024-09-20T19:44:28.081 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (12m) 3m ago 12m 318M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:44:28.081 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (11m) 3m ago 11m 512M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:44:28.081 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (11m) 3m ago 11m 383M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:44:28.081 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (5m) 29s ago 14m 55.4M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:44:28.099 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:27 smithi118 podman[52698]: 2024-09-20 19:44:27.807038641 +0000 UTC m=+0.657575442 container died 53207408a89d3f264c6c94d287ae3f63e58f23b5c950e1fab14ccdcaa760a119 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, ceph=True, distribution-scope=public, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_BRANCH=HEAD, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T19:44:28.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 216649 9.57 MB/sec execute 438 sec latency 978.645 ms 2024-09-20T19:44:28.359 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:28 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:44:28.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:28 smithi116 ceph-mon[63578]: osdmap e128: 8 total, 7 up, 8 in 2024-09-20T19:44:28.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:28 smithi116 ceph-mon[63578]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:28.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:28.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:28.360 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:28.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:28 smithi118 podman[52698]: 2024-09-20 19:44:28.197602382 +0000 UTC m=+1.048139174 container cleanup 53207408a89d3f264c6c94d287ae3f63e58f23b5c950e1fab14ccdcaa760a119 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, version=8, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, distribution-scope=public, io.openshift.expose-services=, ceph=True, release=754) 2024-09-20T19:44:28.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:28 smithi118 bash[52698]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4 2024-09-20T19:44:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:28 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:44:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:28 smithi118 ceph-mon[49197]: osdmap e128: 8 total, 7 up, 8 in 2024-09-20T19:44:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:28 smithi118 ceph-mon[49197]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:28.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:28.523 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 4 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:44:28.524 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:44:28.525 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T19:44:28.525 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 8 2024-09-20T19:44:28.525 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:44:28.525 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:44:28.856 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:28 smithi118 podman[52715]: 2024-09-20 19:44:28.483328996 +0000 UTC m=+0.671249839 container remove 53207408a89d3f264c6c94d287ae3f63e58f23b5c950e1fab14ccdcaa760a119 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, ceph=True, io.openshift.expose-services=, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True) 2024-09-20T19:44:28.936 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:44:28.936 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:44:28.936 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:44:28.936 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:44:28.936 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "10/23 daemons upgraded", 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:44:28.937 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:44:29.148 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:28 smithi118 podman[52798]: 2024-09-20 19:44:28.759206388 +0000 UTC m=+0.018407102 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:44:29.148 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:28 smithi118 podman[52798]: 2024-09-20 19:44:28.885042195 +0000 UTC m=+0.144242897 container create c360eecfb2f5a7801bb268748162ff5eea6a5d73d20368676d45ac56dd4bbd0e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0) 2024-09-20T19:44:29.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 216929 9.58 MB/sec execute 439 sec latency 1014.763 ms 2024-09-20T19:44:29.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:29 smithi116 ceph-mon[63578]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:29.397 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:29 smithi116 ceph-mon[63578]: pgmap v203: 97 pgs: 13 stale+active+clean, 84 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.5 MiB/s wr, 313 op/s; 7.5 KiB/s, 1 objects/s recovering 2024-09-20T19:44:29.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:29 smithi116 ceph-mon[63578]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:29.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:29 smithi116 ceph-mon[63578]: osdmap e129: 8 total, 7 up, 8 in 2024-09-20T19:44:29.398 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:29 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2304673566' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 1 osds down; 5 mgr modules have recently crashed 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout: osd.4 (root=default,host=smithi118) is down 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:44:29.442 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:44:29.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:29 smithi118 ceph-mon[49197]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:29.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:29 smithi118 ceph-mon[49197]: pgmap v203: 97 pgs: 13 stale+active+clean, 84 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.5 MiB/s wr, 313 op/s; 7.5 KiB/s, 1 objects/s recovering 2024-09-20T19:44:29.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:29 smithi118 ceph-mon[49197]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:29.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:29 smithi118 ceph-mon[49197]: osdmap e129: 8 total, 7 up, 8 in 2024-09-20T19:44:29.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:29 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2304673566' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:29.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:29 smithi118 podman[52798]: 2024-09-20 19:44:29.146926288 +0000 UTC m=+0.406127016 container init c360eecfb2f5a7801bb268748162ff5eea6a5d73d20368676d45ac56dd4bbd0e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-09-20T19:44:29.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:29 smithi118 podman[52798]: 2024-09-20 19:44:29.151097129 +0000 UTC m=+0.410297837 container start c360eecfb2f5a7801bb268748162ff5eea6a5d73d20368676d45ac56dd4bbd0e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:44:29.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:29 smithi118 podman[52798]: 2024-09-20 19:44:29.255262487 +0000 UTC m=+0.514463190 container attach c360eecfb2f5a7801bb268748162ff5eea6a5d73d20368676d45ac56dd4bbd0e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-20T19:44:29.466 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:29 smithi118 podman[52798]: 2024-09-20 19:44:29.371102465 +0000 UTC m=+0.630303172 container died c360eecfb2f5a7801bb268748162ff5eea6a5d73d20368676d45ac56dd4bbd0e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:44:30.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 217120 9.56 MB/sec execute 440 sec latency 63.236 ms 2024-09-20T19:44:30.432 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:30 smithi116 ceph-mon[63578]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:30.433 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:30 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/3516957851' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:44:30.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:30 smithi118 ceph-mon[49197]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:44:30.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:30 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/3516957851' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:44:30.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:30 smithi118 podman[52822]: 2024-09-20 19:44:30.302706307 +0000 UTC m=+0.922643868 container remove c360eecfb2f5a7801bb268748162ff5eea6a5d73d20368676d45ac56dd4bbd0e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-20T19:44:30.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:30 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-20T19:44:30.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:30 smithi118 systemd[1]: Stopped Ceph osd.4 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:44:30.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:30 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.4.service: Consumed 3min 39.004s CPU time. 2024-09-20T19:44:30.716 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:30 smithi118 systemd[1]: Starting Ceph osd.4 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:44:31.101 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:30 smithi118 podman[52920]: 2024-09-20 19:44:30.666673857 +0000 UTC m=+0.020664359 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:44:31.101 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:30 smithi118 podman[52920]: 2024-09-20 19:44:30.812527401 +0000 UTC m=+0.166517911 container create 5e76722d63e2bbcee53ad720c4396177cd10947ca927b74e4e47ea4a96e09dfd (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:44:31.101 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:31 smithi118 podman[52920]: 2024-09-20 19:44:31.09971632 +0000 UTC m=+0.453706826 container init 5e76722d63e2bbcee53ad720c4396177cd10947ca927b74e4e47ea4a96e09dfd (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-20T19:44:31.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 217384 9.55 MB/sec execute 441 sec latency 57.702 ms 2024-09-20T19:44:31.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:31 smithi118 ceph-mon[49197]: pgmap v205: 97 pgs: 1 active+undersized, 15 peering, 6 stale+active+clean, 6 active+undersized+degraded, 69 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.2 MiB/s wr, 432 op/s; 56/1548 objects degraded (3.618%) 2024-09-20T19:44:31.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:31 smithi118 ceph-mon[49197]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-20T19:44:31.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:31 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 56/1548 objects degraded (3.618%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:31.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:31 smithi118 podman[52920]: 2024-09-20 19:44:31.103593054 +0000 UTC m=+0.457583556 container start 5e76722d63e2bbcee53ad720c4396177cd10947ca927b74e4e47ea4a96e09dfd (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-20T19:44:31.466 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:31 smithi118 podman[52920]: 2024-09-20 19:44:31.191644536 +0000 UTC m=+0.545635049 container attach 5e76722d63e2bbcee53ad720c4396177cd10947ca927b74e4e47ea4a96e09dfd (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:44:31.466 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:31 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:44:31.466 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:31 smithi118 bash[52920]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:44:31.466 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:31 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:44:31.466 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:31 smithi118 bash[52920]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:44:31.503 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:31 smithi116 ceph-mon[63578]: pgmap v205: 97 pgs: 1 active+undersized, 15 peering, 6 stale+active+clean, 6 active+undersized+degraded, 69 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.2 MiB/s wr, 432 op/s; 56/1548 objects degraded (3.618%) 2024-09-20T19:44:31.503 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:31 smithi116 ceph-mon[63578]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-20T19:44:31.503 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:31 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 56/1548 objects degraded (3.618%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:32.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 217612 9.54 MB/sec execute 442 sec latency 78.245 ms 2024-09-20T19:44:32.931 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T19:44:32.931 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 bash[52920]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T19:44:32.931 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T19:44:32.932 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 bash[52920]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-20T19:44:32.932 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T19:44:32.932 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 bash[52920]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 bash[52920]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 bash[52920]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 bash[52920]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate[52935]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 bash[52920]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-20T19:44:33.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:32 smithi118 podman[52920]: 2024-09-20 19:44:32.961021858 +0000 UTC m=+2.315012373 container died 5e76722d63e2bbcee53ad720c4396177cd10947ca927b74e4e47ea4a96e09dfd (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:44:33.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 217771 9.52 MB/sec execute 443 sec latency 69.568 ms 2024-09-20T19:44:33.582 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:33 smithi116 ceph-mon[63578]: pgmap v206: 97 pgs: 4 active+undersized, 15 peering, 5 stale+active+clean, 9 active+undersized+degraded, 64 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.0 MiB/s wr, 197 op/s; 79/1542 objects degraded (5.123%) 2024-09-20T19:44:33.640 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:33 smithi118 ceph-mon[49197]: pgmap v206: 97 pgs: 4 active+undersized, 15 peering, 5 stale+active+clean, 9 active+undersized+degraded, 64 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.0 MiB/s wr, 197 op/s; 79/1542 objects degraded (5.123%) 2024-09-20T19:44:33.899 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:33 smithi118 podman[53084]: 2024-09-20 19:44:33.638948762 +0000 UTC m=+0.671643157 container remove 5e76722d63e2bbcee53ad720c4396177cd10947ca927b74e4e47ea4a96e09dfd (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-20T19:44:33.899 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:33 smithi118 podman[53144]: 2024-09-20 19:44:33.752994141 +0000 UTC m=+0.017334525 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:44:33.899 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:33 smithi118 podman[53144]: 2024-09-20 19:44:33.898595964 +0000 UTC m=+0.162936349 container create 4ae34596cfeaa8eca96b49568232123203453026caa52c7516a5074b8c4b416e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-20T19:44:34.170 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:34 smithi118 podman[53144]: 2024-09-20 19:44:34.142960775 +0000 UTC m=+0.407301165 container init 4ae34596cfeaa8eca96b49568232123203453026caa52c7516a5074b8c4b416e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-20T19:44:34.170 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:34 smithi118 podman[53144]: 2024-09-20 19:44:34.146567035 +0000 UTC m=+0.410907422 container start 4ae34596cfeaa8eca96b49568232123203453026caa52c7516a5074b8c4b416e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:44:34.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 217964 9.52 MB/sec execute 444 sec latency 59.823 ms 2024-09-20T19:44:34.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:34 smithi118 ceph-osd[53163]: -- 172.21.15.118:0/1356710522 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5651cf10c1e0 con 0x5651cf0a0000 2024-09-20T19:44:34.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:34 smithi118 bash[53144]: 4ae34596cfeaa8eca96b49568232123203453026caa52c7516a5074b8c4b416e 2024-09-20T19:44:34.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:34 smithi118 systemd[1]: Started Ceph osd.4 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:44:35.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 218321 9.52 MB/sec execute 445 sec latency 59.153 ms 2024-09-20T19:44:35.378 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:35 smithi116 ceph-mon[63578]: pgmap v207: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.7 MiB/s wr, 375 op/s; 207/1632 objects degraded (12.684%) 2024-09-20T19:44:35.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:35.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:35.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:44:35.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:35 smithi116 ceph-mon[63578]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-20T19:44:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:35 smithi118 ceph-mon[49197]: pgmap v207: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.7 MiB/s wr, 375 op/s; 207/1632 objects degraded (12.684%) 2024-09-20T19:44:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:44:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:35 smithi118 ceph-mon[49197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-20T19:44:36.083 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:35 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:44:35.780+0000 7f856ba1d640 -1 Falling back to public interface 2024-09-20T19:44:36.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 218784 9.53 MB/sec execute 446 sec latency 51.229 ms 2024-09-20T19:44:36.415 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:36 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 205/1620 objects degraded (12.654%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:36.689 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:36 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 205/1620 objects degraded (12.654%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:37.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 219176 9.52 MB/sec execute 447 sec latency 44.616 ms 2024-09-20T19:44:37.421 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:37 smithi116 ceph-mon[63578]: pgmap v208: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.2 MiB/s wr, 343 op/s; 205/1620 objects degraded (12.654%) 2024-09-20T19:44:37.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:37 smithi118 ceph-mon[49197]: pgmap v208: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.2 MiB/s wr, 343 op/s; 205/1620 objects degraded (12.654%) 2024-09-20T19:44:38.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 219374 9.51 MB/sec execute 448 sec latency 88.028 ms 2024-09-20T19:44:39.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 219662 9.51 MB/sec execute 449 sec latency 64.487 ms 2024-09-20T19:44:39.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:39 smithi116 ceph-mon[63578]: pgmap v209: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.4 MiB/s wr, 319 op/s; 211/1692 objects degraded (12.470%) 2024-09-20T19:44:39.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:39.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:39.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:39.528 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:39 smithi118 ceph-mon[49197]: pgmap v209: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.4 MiB/s wr, 319 op/s; 211/1692 objects degraded (12.470%) 2024-09-20T19:44:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:40.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 219899 9.50 MB/sec execute 450 sec latency 85.457 ms 2024-09-20T19:44:41.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 220124 9.50 MB/sec execute 451 sec latency 75.602 ms 2024-09-20T19:44:41.602 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:41 smithi116 ceph-mon[63578]: pgmap v210: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.6 MiB/s wr, 356 op/s; 200/1653 objects degraded (12.099%) 2024-09-20T19:44:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:41 smithi118 ceph-mon[49197]: pgmap v210: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.6 MiB/s wr, 356 op/s; 200/1653 objects degraded (12.099%) 2024-09-20T19:44:42.177 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:41 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:44:41.867+0000 7f856ba1d640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-20T19:44:42.307 INFO:teuthology.orchestra.run.smithi116.stdout: 5 220347 9.50 MB/sec execute 452 sec latency 64.520 ms 2024-09-20T19:44:42.662 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:42 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 202/1668 objects degraded (12.110%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:42.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:42 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 202/1668 objects degraded (12.110%), 24 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:43.215 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:42 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:44:42.909+0000 7f856ba1d640 -1 osd.4 127 log_to_monitors true 2024-09-20T19:44:43.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 220596 9.50 MB/sec execute 453 sec latency 81.327 ms 2024-09-20T19:44:43.588 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: pgmap v211: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.4 MiB/s wr, 335 op/s; 202/1668 objects degraded (12.110%) 2024-09-20T19:44:43.588 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.588 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='osd.4 [v2:172.21.15.118:6800/3366330484,v1:172.21.15.118:6801/3366330484]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:43.589 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:43.590 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: pgmap v211: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.4 MiB/s wr, 335 op/s; 202/1668 objects degraded (12.110%) 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='osd.4 [v2:172.21.15.118:6800/3366330484,v1:172.21.15.118:6801/3366330484]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.675 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:43.676 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:44:43.676 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:43.676 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:43.676 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:44:43.676 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T19:44:43.965 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:44:43 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:44:43.586+0000 7f85637c7640 -1 osd.4 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:44:44.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 220970 9.49 MB/sec execute 454 sec latency 57.052 ms 2024-09-20T19:44:44.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:44 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T19:44:44.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:44 smithi118 ceph-mon[49197]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-20T19:44:44.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:44 smithi118 ceph-mon[49197]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T19:44:44.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:44 smithi118 ceph-mon[49197]: from='osd.4 [v2:172.21.15.118:6800/3366330484,v1:172.21.15.118:6801/3366330484]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:44:44.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:44 smithi118 ceph-mon[49197]: osdmap e130: 8 total, 7 up, 8 in 2024-09-20T19:44:44.699 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:44 smithi118 ceph-mon[49197]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:44:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:44 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T19:44:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:44 smithi116 ceph-mon[63578]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-20T19:44:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:44 smithi116 ceph-mon[63578]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T19:44:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:44 smithi116 ceph-mon[63578]: from='osd.4 [v2:172.21.15.118:6800/3366330484,v1:172.21.15.118:6801/3366330484]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:44:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:44 smithi116 ceph-mon[63578]: osdmap e130: 8 total, 7 up, 8 in 2024-09-20T19:44:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:44 smithi116 ceph-mon[63578]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:44:45.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 221049 9.47 MB/sec execute 455 sec latency 727.476 ms 2024-09-20T19:44:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:45 smithi118 ceph-mon[49197]: pgmap v213: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 192/1599 objects degraded (12.008%) 2024-09-20T19:44:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:45 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:44:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:45 smithi118 ceph-mon[49197]: osd.4 [v2:172.21.15.118:6800/3366330484,v1:172.21.15.118:6801/3366330484] boot 2024-09-20T19:44:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:45 smithi118 ceph-mon[49197]: osdmap e131: 8 total, 8 up, 8 in 2024-09-20T19:44:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:45 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T19:44:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:45 smithi116 ceph-mon[63578]: pgmap v213: 97 pgs: 13 active+undersized, 24 active+undersized+degraded, 60 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 192/1599 objects degraded (12.008%) 2024-09-20T19:44:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:45 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:44:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:45 smithi116 ceph-mon[63578]: osd.4 [v2:172.21.15.118:6800/3366330484,v1:172.21.15.118:6801/3366330484] boot 2024-09-20T19:44:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:45 smithi116 ceph-mon[63578]: osdmap e131: 8 total, 8 up, 8 in 2024-09-20T19:44:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:45 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T19:44:46.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 221049 9.45 MB/sec execute 456 sec latency 1727.530 ms 2024-09-20T19:44:46.862 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:46 smithi116 ceph-mon[63578]: osdmap e132: 8 total, 8 up, 8 in 2024-09-20T19:44:46.863 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:46 smithi116 ceph-mon[63578]: pgmap v216: 97 pgs: 5 peering, 11 active+undersized, 21 active+undersized+degraded, 60 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 376 op/s; 169/1617 objects degraded (10.451%) 2024-09-20T19:44:46.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:46 smithi118 ceph-mon[49197]: osdmap e132: 8 total, 8 up, 8 in 2024-09-20T19:44:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:46 smithi118 ceph-mon[49197]: pgmap v216: 97 pgs: 5 peering, 11 active+undersized, 21 active+undersized+degraded, 60 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 376 op/s; 169/1617 objects degraded (10.451%) 2024-09-20T19:44:47.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 221176 9.44 MB/sec execute 457 sec latency 2100.217 ms 2024-09-20T19:44:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:47 smithi116 ceph-mon[63578]: osdmap e133: 8 total, 8 up, 8 in 2024-09-20T19:44:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:47 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 169/1617 objects degraded (10.451%), 21 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:47 smithi116 ceph-mon[63578]: osdmap e134: 8 total, 8 up, 8 in 2024-09-20T19:44:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:47 smithi118 ceph-mon[49197]: osdmap e133: 8 total, 8 up, 8 in 2024-09-20T19:44:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:47 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 169/1617 objects degraded (10.451%), 21 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:47 smithi118 ceph-mon[49197]: osdmap e134: 8 total, 8 up, 8 in 2024-09-20T19:44:48.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 221212 9.42 MB/sec execute 458 sec latency 918.745 ms 2024-09-20T19:44:49.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 221569 9.42 MB/sec execute 459 sec latency 1025.973 ms 2024-09-20T19:44:49.446 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:49 smithi116 ceph-mon[63578]: pgmap v219: 97 pgs: 5 peering, 1 active+recovery_wait+degraded, 10 active+undersized, 18 active+undersized+degraded, 63 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 MiB/s wr, 135 op/s; 153/1590 objects degraded (9.623%); 3.0 KiB/s, 1 objects/s recovering 2024-09-20T19:44:49.446 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:49 smithi116 ceph-mon[63578]: osdmap e135: 8 total, 8 up, 8 in 2024-09-20T19:44:49.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:49 smithi118 ceph-mon[49197]: pgmap v219: 97 pgs: 5 peering, 1 active+recovery_wait+degraded, 10 active+undersized, 18 active+undersized+degraded, 63 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 MiB/s wr, 135 op/s; 153/1590 objects degraded (9.623%); 3.0 KiB/s, 1 objects/s recovering 2024-09-20T19:44:49.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:49 smithi118 ceph-mon[49197]: osdmap e135: 8 total, 8 up, 8 in 2024-09-20T19:44:50.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 221768 9.41 MB/sec execute 460 sec latency 880.868 ms 2024-09-20T19:44:50.480 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:50 smithi116 ceph-mon[63578]: osdmap e136: 8 total, 8 up, 8 in 2024-09-20T19:44:50.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:50 smithi118 ceph-mon[49197]: osdmap e136: 8 total, 8 up, 8 in 2024-09-20T19:44:51.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 222064 9.42 MB/sec execute 461 sec latency 1016.229 ms 2024-09-20T19:44:51.523 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:51 smithi116 ceph-mon[63578]: pgmap v222: 97 pgs: 1 active+recovery_wait, 7 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.7 MiB/s wr, 91 op/s; 16/1572 objects degraded (1.018%); 2.4 MiB/s, 43 objects/s recovering 2024-09-20T19:44:51.523 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:51 smithi116 ceph-mon[63578]: osdmap e137: 8 total, 8 up, 8 in 2024-09-20T19:44:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:51 smithi118 ceph-mon[49197]: pgmap v222: 97 pgs: 1 active+recovery_wait, 7 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 87 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.7 MiB/s wr, 91 op/s; 16/1572 objects degraded (1.018%); 2.4 MiB/s, 43 objects/s recovering 2024-09-20T19:44:51.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:51 smithi118 ceph-mon[49197]: osdmap e137: 8 total, 8 up, 8 in 2024-09-20T19:44:52.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 222298 9.40 MB/sec execute 462 sec latency 63.172 ms 2024-09-20T19:44:52.573 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:52 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 16/1572 objects degraded (1.018%), 2 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:52.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:52 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 16/1572 objects degraded (1.018%), 2 pgs degraded (PG_DEGRADED) 2024-09-20T19:44:53.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 222504 9.40 MB/sec execute 463 sec latency 127.905 ms 2024-09-20T19:44:53.597 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:53 smithi116 ceph-mon[63578]: pgmap v224: 97 pgs: 1 active+recovery_wait, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 92 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s wr, 207 op/s; 16/1560 objects degraded (1.026%); 2.8 MiB/s, 52 keys/s, 42 objects/s recovering 2024-09-20T19:44:53.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:53 smithi118 ceph-mon[49197]: pgmap v224: 97 pgs: 1 active+recovery_wait, 2 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 92 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.8 MiB/s wr, 207 op/s; 16/1560 objects degraded (1.026%); 2.8 MiB/s, 52 keys/s, 42 objects/s recovering 2024-09-20T19:44:54.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 222704 9.38 MB/sec execute 464 sec latency 90.667 ms 2024-09-20T19:44:54.611 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:54 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1560 objects degraded (1.026%), 2 pgs degraded) 2024-09-20T19:44:54.938 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:54 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1560 objects degraded (1.026%), 2 pgs degraded) 2024-09-20T19:44:55.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 223158 9.38 MB/sec execute 465 sec latency 53.911 ms 2024-09-20T19:44:55.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:55 smithi116 ceph-mon[63578]: pgmap v225: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.0 MiB/s wr, 382 op/s; 2.0 MiB/s, 37 keys/s, 33 objects/s recovering 2024-09-20T19:44:55.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:55 smithi118 ceph-mon[49197]: pgmap v225: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.0 MiB/s wr, 382 op/s; 2.0 MiB/s, 37 keys/s, 33 objects/s recovering 2024-09-20T19:44:56.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 223513 9.38 MB/sec execute 466 sec latency 104.934 ms 2024-09-20T19:44:57.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 223835 9.39 MB/sec execute 467 sec latency 51.603 ms 2024-09-20T19:44:57.649 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:57 smithi116 ceph-mon[63578]: pgmap v226: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 3.2 MiB/s wr, 349 op/s; 591 KiB/s, 30 keys/s, 17 objects/s recovering 2024-09-20T19:44:57.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:57 smithi118 ceph-mon[49197]: pgmap v226: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 3.2 MiB/s wr, 349 op/s; 591 KiB/s, 30 keys/s, 17 objects/s recovering 2024-09-20T19:44:58.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 224136 9.39 MB/sec execute 468 sec latency 77.119 ms 2024-09-20T19:44:58.713 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:58.714 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:58.714 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:58 smithi116 ceph-mon[63578]: pgmap v227: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.1 MiB/s wr, 327 op/s; 336 KiB/s, 27 keys/s, 4 objects/s recovering 2024-09-20T19:44:58.714 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T19:44:58.714 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:58 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T19:44:58.714 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:44:58 smithi116 ceph-mon[63578]: Upgrade: osd.5 is safe to restart 2024-09-20T19:44:58.914 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:44:58.914 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:44:58.914 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:58 smithi118 ceph-mon[49197]: pgmap v227: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.1 MiB/s wr, 327 op/s; 336 KiB/s, 27 keys/s, 4 objects/s recovering 2024-09-20T19:44:58.914 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T19:44:58.914 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:58 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T19:44:58.914 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:44:58 smithi118 ceph-mon[49197]: Upgrade: osd.5 is safe to restart 2024-09-20T19:44:59.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 224383 9.38 MB/sec execute 469 sec latency 64.428 ms 2024-09-20T19:44:59.883 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:45:00.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 224600 9.37 MB/sec execute 470 sec latency 82.498 ms 2024-09-20T19:45:00.635 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (7m) 61s ago 16m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (4m) 61s ago 16m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (4m) 22s ago 15m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (6m) 61s ago 16m 47.1M - dad864ee21e9 8a163a77164a 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (10m) 61s ago 10m 37.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (10m) 22s ago 10m 14.9M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (6m) 61s ago 17m 497M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (5m) 22s ago 15m 450M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (5m) 61s ago 17m 51.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (5m) 22s ago 15m 43.5M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (10m) 61s ago 10m 290M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:45:00.636 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (9m) 22s ago 9m 86.1M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (7m) 61s ago 16m 10.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (6m) 22s ago 15m 11.0M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (3m) 61s ago 14m 343M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (2m) 61s ago 14m 255M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (119s) 61s ago 13m 214M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (71s) 61s ago 13m 14.3M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (26s) 22s ago 13m 11.4M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (12m) 22s ago 12m 537M 2704M 17.2.0 e1d6a67b021e 7ddae2b29669 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (12m) 22s ago 12m 464M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (11m) 22s ago 11m 686M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:45:00.637 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (5m) 61s ago 15m 55.4M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: pgmap v228: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 281 KiB/s, 22 keys/s, 3 objects/s recovering 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: Upgrade: Updating osd.5 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: Deploying daemon osd.5 on smithi118 2024-09-20T19:45:01.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:00 smithi116 ceph-mon[63578]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-20T19:45:01.080 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 5 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 9 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:45:01.081 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: pgmap v228: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 281 KiB/s, 22 keys/s, 3 objects/s recovering 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: Upgrade: Updating osd.5 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: Deploying daemon osd.5 on smithi118 2024-09-20T19:45:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:00 smithi118 ceph-mon[49197]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:01.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 224766 9.36 MB/sec execute 471 sec latency 126.316 ms 2024-09-20T19:45:01.455 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:45:01.455 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:45:01.455 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:45:01.455 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:45:01.455 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:45:01.455 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:45:01.456 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:45:01.456 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:45:01.456 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:45:01.456 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "11/23 daemons upgraded", 2024-09-20T19:45:01.456 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:45:01.456 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:45:01.456 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:45:01.929 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:45:01.929 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:45:01.929 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:45:01.930 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:45:01.930 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:45:01.930 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:45:01.930 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:45:01.930 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:45:01.930 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:45:02.075 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:01 smithi116 ceph-mon[63578]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:02.075 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:01 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/3235750318' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:02.075 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:01 smithi116 ceph-mon[63578]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:02.075 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:01 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/4096298683' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:45:02.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 225034 9.36 MB/sec execute 472 sec latency 67.560 ms 2024-09-20T19:45:02.374 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:01 smithi118 ceph-mon[49197]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:02.374 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:01 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/3235750318' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:02.374 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:01 smithi118 ceph-mon[49197]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:02.374 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:01 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/4096298683' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:45:03.132 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:02 smithi116 ceph-mon[63578]: pgmap v229: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 337 op/s; 9.3 KiB/s, 1 objects/s recovering 2024-09-20T19:45:03.133 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:02 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:03.184 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:02 smithi118 ceph-mon[49197]: pgmap v229: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 337 op/s; 9.3 KiB/s, 1 objects/s recovering 2024-09-20T19:45:03.184 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:02 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:03.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 225267 9.35 MB/sec execute 473 sec latency 43.269 ms 2024-09-20T19:45:03.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:03 smithi118 systemd[1]: Stopping Ceph osd.5 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:45:03.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[32862]: 2024-09-20T19:45:03.529+0000 7f8fa0b42700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:45:03.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[32862]: 2024-09-20T19:45:03.529+0000 7f8fa0b42700 -1 osd.5 137 *** Got signal Terminated *** 2024-09-20T19:45:03.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[32862]: 2024-09-20T19:45:03.529+0000 7f8fa0b42700 -1 osd.5 137 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:45:04.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:03 smithi116 ceph-mon[63578]: osd.5 marked itself down and dead 2024-09-20T19:45:04.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 225489 9.35 MB/sec execute 474 sec latency 309.187 ms 2024-09-20T19:45:04.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:03 smithi118 ceph-mon[49197]: osd.5 marked itself down and dead 2024-09-20T19:45:04.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:04 smithi118 podman[54710]: 2024-09-20 19:45:04.647394096 +0000 UTC m=+1.244343096 container died 7ddae2b29669aa1a0c8bdc314835db828746a25f74e7e59f625b6c18d4b1cff2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, architecture=x86_64, io.buildah.version=1.19.8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, release=754, vcs-type=git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-20T19:45:05.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:05 smithi116 ceph-mon[63578]: pgmap v230: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 8.8 KiB/s, 1 objects/s recovering 2024-09-20T19:45:05.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:05 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:45:05.264 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:05 smithi116 ceph-mon[63578]: osdmap e138: 8 total, 7 up, 8 in 2024-09-20T19:45:05.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 225614 9.34 MB/sec execute 475 sec latency 1044.220 ms 2024-09-20T19:45:05.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:05 smithi118 ceph-mon[49197]: pgmap v230: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 8.8 KiB/s, 1 objects/s recovering 2024-09-20T19:45:05.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:05 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:45:05.398 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:05 smithi118 ceph-mon[49197]: osdmap e138: 8 total, 7 up, 8 in 2024-09-20T19:45:05.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:05 smithi118 podman[54710]: 2024-09-20 19:45:05.397124741 +0000 UTC m=+1.994073778 container cleanup 7ddae2b29669aa1a0c8bdc314835db828746a25f74e7e59f625b6c18d4b1cff2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, vendor=Red Hat, Inc., distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, version=8, ceph=True) 2024-09-20T19:45:05.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:05 smithi118 bash[54710]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5 2024-09-20T19:45:06.025 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:05 smithi118 podman[54725]: 2024-09-20 19:45:05.724050657 +0000 UTC m=+1.073980133 container remove 7ddae2b29669aa1a0c8bdc314835db828746a25f74e7e59f625b6c18d4b1cff2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, io.openshift.tags=base centos centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.expose-services=, name=centos-stream, RELEASE=HEAD, ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, version=8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-20T19:45:06.038 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:06 smithi116 ceph-mon[63578]: osdmap e139: 8 total, 7 up, 8 in 2024-09-20T19:45:06.282 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:06 smithi118 ceph-mon[49197]: osdmap e139: 8 total, 7 up, 8 in 2024-09-20T19:45:06.282 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:06 smithi118 podman[54814]: 2024-09-20 19:45:06.028869741 +0000 UTC m=+0.020607188 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:45:06.282 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:06 smithi118 podman[54814]: 2024-09-20 19:45:06.150341043 +0000 UTC m=+0.142078491 container create ebae71ae1a78c14f34be2e46cc3d0b3d41ef9264fd1ccb120914d7bbb0845017 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-20T19:45:06.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 225886 9.34 MB/sec execute 476 sec latency 49.898 ms 2024-09-20T19:45:06.637 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:06 smithi118 podman[54814]: 2024-09-20 19:45:06.403729195 +0000 UTC m=+0.395466654 container init ebae71ae1a78c14f34be2e46cc3d0b3d41ef9264fd1ccb120914d7bbb0845017 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True) 2024-09-20T19:45:06.638 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:06 smithi118 podman[54814]: 2024-09-20 19:45:06.408767397 +0000 UTC m=+0.400504852 container start ebae71ae1a78c14f34be2e46cc3d0b3d41ef9264fd1ccb120914d7bbb0845017 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:45:06.638 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:06 smithi118 podman[54814]: 2024-09-20 19:45:06.512194924 +0000 UTC m=+0.503932379 container attach ebae71ae1a78c14f34be2e46cc3d0b3d41ef9264fd1ccb120914d7bbb0845017 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:45:06.964 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:06 smithi118 podman[54814]: 2024-09-20 19:45:06.636614508 +0000 UTC m=+0.628351966 container died ebae71ae1a78c14f34be2e46cc3d0b3d41ef9264fd1ccb120914d7bbb0845017 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:45:07.050 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:07 smithi116 ceph-mon[63578]: pgmap v233: 97 pgs: 2 active+undersized, 11 stale+active+clean, 3 active+undersized+degraded, 81 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 404 op/s; 19/1626 objects degraded (1.169%) 2024-09-20T19:45:07.051 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:07 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 19/1626 objects degraded (1.169%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:07.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 226177 9.32 MB/sec execute 477 sec latency 52.235 ms 2024-09-20T19:45:07.461 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:07 smithi118 ceph-mon[49197]: pgmap v233: 97 pgs: 2 active+undersized, 11 stale+active+clean, 3 active+undersized+degraded, 81 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 404 op/s; 19/1626 objects degraded (1.169%) 2024-09-20T19:45:07.461 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:07 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 19/1626 objects degraded (1.169%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:07.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:07 smithi118 podman[54836]: 2024-09-20 19:45:07.460001808 +0000 UTC m=+0.816758882 container remove ebae71ae1a78c14f34be2e46cc3d0b3d41ef9264fd1ccb120914d7bbb0845017 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:45:07.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:07 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-20T19:45:07.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:07 smithi118 systemd[1]: Stopped Ceph osd.5 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:45:07.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:07 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.5.service: Consumed 4min 32.973s CPU time. 2024-09-20T19:45:07.716 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:07 smithi118 systemd[1]: Starting Ceph osd.5 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:45:08.198 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:07 smithi118 podman[54935]: 2024-09-20 19:45:07.796758393 +0000 UTC m=+0.015722442 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:45:08.198 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:07 smithi118 podman[54935]: 2024-09-20 19:45:07.919618588 +0000 UTC m=+0.138582624 container create f2d0dd13394c90887aff67669d4c020e657c10785d608322ad8f4d8da52c1da1 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-20T19:45:08.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 226517 9.32 MB/sec execute 478 sec latency 63.661 ms 2024-09-20T19:45:08.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:08 smithi118 podman[54935]: 2024-09-20 19:45:08.197332082 +0000 UTC m=+0.416296117 container init f2d0dd13394c90887aff67669d4c020e657c10785d608322ad8f4d8da52c1da1 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:45:08.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:08 smithi118 podman[54935]: 2024-09-20 19:45:08.201411492 +0000 UTC m=+0.420375530 container start f2d0dd13394c90887aff67669d4c020e657c10785d608322ad8f4d8da52c1da1 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-20T19:45:08.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:08 smithi118 podman[54935]: 2024-09-20 19:45:08.33112545 +0000 UTC m=+0.550089503 container attach f2d0dd13394c90887aff67669d4c020e657c10785d608322ad8f4d8da52c1da1 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-20T19:45:08.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:08.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:08 smithi118 bash[54935]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:08.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:08.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:08 smithi118 bash[54935]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:09.115 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:09 smithi116 ceph-mon[63578]: pgmap v234: 97 pgs: 2 active+undersized, 7 stale+active+clean, 12 active+undersized+degraded, 76 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 451 op/s; 85/1629 objects degraded (5.218%) 2024-09-20T19:45:09.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 226838 9.31 MB/sec execute 479 sec latency 46.472 ms 2024-09-20T19:45:09.427 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-mon[49197]: pgmap v234: 97 pgs: 2 active+undersized, 7 stale+active+clean, 12 active+undersized+degraded, 76 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 451 op/s; 85/1629 objects degraded (5.218%) 2024-09-20T19:45:09.711 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T19:45:09.711 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 bash[54935]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T19:45:09.711 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T19:45:09.711 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 bash[54935]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-20T19:45:09.711 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T19:45:09.711 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 bash[54935]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-20T19:45:09.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T19:45:09.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 bash[54935]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T19:45:09.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-20T19:45:09.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 bash[54935]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-20T19:45:09.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T19:45:09.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 bash[54935]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T19:45:09.966 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate[54950]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-20T19:45:09.966 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 bash[54935]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-20T19:45:09.966 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:09 smithi118 podman[54935]: 2024-09-20 19:45:09.738781018 +0000 UTC m=+1.957745065 container died f2d0dd13394c90887aff67669d4c020e657c10785d608322ad8f4d8da52c1da1 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:45:10.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 227130 9.31 MB/sec execute 480 sec latency 69.697 ms 2024-09-20T19:45:10.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:10 smithi118 podman[55081]: 2024-09-20 19:45:10.711526406 +0000 UTC m=+0.966796006 container remove f2d0dd13394c90887aff67669d4c020e657c10785d608322ad8f4d8da52c1da1 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-activate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T19:45:10.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:10 smithi118 podman[55142]: 2024-09-20 19:45:10.823158654 +0000 UTC m=+0.019504867 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:45:10.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:10 smithi118 podman[55142]: 2024-09-20 19:45:10.94606081 +0000 UTC m=+0.142407004 container create e6a039aa388ff69fb822691eb29312ebea5fb825e68d173f2536d59da317020a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-20T19:45:11.138 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:11 smithi116 ceph-mon[63578]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 382 op/s; 184/1635 objects degraded (11.254%) 2024-09-20T19:45:11.308 INFO:teuthology.orchestra.run.smithi116.stdout: 5 227485 9.33 MB/sec execute 481 sec latency 100.554 ms 2024-09-20T19:45:11.333 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:11 smithi118 ceph-mon[49197]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 382 op/s; 184/1635 objects degraded (11.254%) 2024-09-20T19:45:11.333 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:11 smithi118 podman[55142]: 2024-09-20 19:45:11.232244197 +0000 UTC m=+0.428590415 container init e6a039aa388ff69fb822691eb29312ebea5fb825e68d173f2536d59da317020a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-20T19:45:11.333 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:11 smithi118 podman[55142]: 2024-09-20 19:45:11.236183092 +0000 UTC m=+0.432529287 container start e6a039aa388ff69fb822691eb29312ebea5fb825e68d173f2536d59da317020a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:45:11.333 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:11 smithi118 ceph-osd[55162]: -- 172.21.15.118:0/2205927180 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5559a4f441e0 con 0x5559a4f2a000 2024-09-20T19:45:11.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:11 smithi118 bash[55142]: e6a039aa388ff69fb822691eb29312ebea5fb825e68d173f2536d59da317020a 2024-09-20T19:45:11.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:11 smithi118 systemd[1]: Started Ceph osd.5 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:45:12.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 227905 9.32 MB/sec execute 482 sec latency 41.880 ms 2024-09-20T19:45:12.407 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:12.407 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:12.408 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:45:12.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:12.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:12.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:45:13.215 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:12 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:45:12.863+0000 7fed0433a640 -1 Falling back to public interface 2024-09-20T19:45:13.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 228166 9.32 MB/sec execute 483 sec latency 82.567 ms 2024-09-20T19:45:13.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:13 smithi116 ceph-mon[63578]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 MiB/s wr, 215 op/s; 183/1650 objects degraded (11.091%) 2024-09-20T19:45:13.457 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:13 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 183/1650 objects degraded (11.091%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:13.458 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:13.458 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:13.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:13 smithi118 ceph-mon[49197]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 MiB/s wr, 215 op/s; 183/1650 objects degraded (11.091%) 2024-09-20T19:45:13.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:13 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 183/1650 objects degraded (11.091%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:13.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:13.527 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:14.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 228349 9.31 MB/sec execute 484 sec latency 53.958 ms 2024-09-20T19:45:15.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 228637 9.31 MB/sec execute 485 sec latency 60.361 ms 2024-09-20T19:45:15.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:15 smithi116 ceph-mon[63578]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 189/1674 objects degraded (11.290%) 2024-09-20T19:45:15.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:15 smithi118 ceph-mon[49197]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.8 MiB/s wr, 370 op/s; 189/1674 objects degraded (11.290%) 2024-09-20T19:45:16.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 228913 9.30 MB/sec execute 486 sec latency 79.326 ms 2024-09-20T19:45:17.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 229203 9.31 MB/sec execute 487 sec latency 69.346 ms 2024-09-20T19:45:17.602 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:17 smithi116 ceph-mon[63578]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 458 op/s; 190/1680 objects degraded (11.310%) 2024-09-20T19:45:17.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:17 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:17 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:17 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:17 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.603 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:17 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 190/1680 objects degraded (11.310%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:17.689 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:17 smithi118 ceph-mon[49197]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 458 op/s; 190/1680 objects degraded (11.310%) 2024-09-20T19:45:17.689 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:17 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.689 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:17 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.689 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:17 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.689 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:17 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:17.689 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:17 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 190/1680 objects degraded (11.310%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:18.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 229485 9.30 MB/sec execute 488 sec latency 79.314 ms 2024-09-20T19:45:19.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 229659 9.29 MB/sec execute 489 sec latency 82.932 ms 2024-09-20T19:45:19.345 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:45:19.036+0000 7fed0433a640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-20T19:45:19.430 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:19 smithi116 ceph-mon[63578]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 429 op/s; 185/1695 objects degraded (10.914%) 2024-09-20T19:45:19.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:19 smithi118 ceph-mon[49197]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 429 op/s; 185/1695 objects degraded (10.914%) 2024-09-20T19:45:20.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 229867 9.28 MB/sec execute 490 sec latency 68.724 ms 2024-09-20T19:45:20.463 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:45:20.191+0000 7fed0433a640 -1 osd.5 137 log_to_monitors true 2024-09-20T19:45:20.702 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:20 smithi116 ceph-mon[63578]: from='osd.5 [v2:172.21.15.118:6808/2495967899,v1:172.21.15.118:6809/2495967899]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T19:45:20.702 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:20 smithi116 ceph-mon[63578]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T19:45:20.714 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:20 smithi118 ceph-mon[49197]: from='osd.5 [v2:172.21.15.118:6808/2495967899,v1:172.21.15.118:6809/2495967899]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T19:45:20.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:20 smithi118 ceph-mon[49197]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T19:45:21.214 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:45:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:45:20.770+0000 7fecf80dc640 -1 osd.5 137 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:45:21.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 230163 9.27 MB/sec execute 491 sec latency 108.040 ms 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 352 op/s; 182/1650 objects degraded (11.030%) 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: osdmap e140: 8 total, 7 up, 8 in 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='osd.5 [v2:172.21.15.118:6808/2495967899,v1:172.21.15.118:6809/2495967899]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='osd.5 ' entity='osd.5' 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:45:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:45:21.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:21.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:21 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 352 op/s; 182/1650 objects degraded (11.030%) 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: osdmap e140: 8 total, 7 up, 8 in 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='osd.5 [v2:172.21.15.118:6808/2495967899,v1:172.21.15.118:6809/2495967899]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='osd.5 ' entity='osd.5' 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:21.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:45:21.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:45:21.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:21.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:21 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:22.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 230200 9.25 MB/sec execute 492 sec latency 865.840 ms 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: osd.5 [v2:172.21.15.118:6808/2495967899,v1:172.21.15.118:6809/2495967899] boot 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: osdmap e141: 8 total, 8 up, 8 in 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 182/1650 objects degraded (11.030%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:22.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:22 smithi116 ceph-mon[63578]: osdmap e142: 8 total, 8 up, 8 in 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: osd.5 [v2:172.21.15.118:6808/2495967899,v1:172.21.15.118:6809/2495967899] boot 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: osdmap e141: 8 total, 8 up, 8 in 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 182/1650 objects degraded (11.030%), 23 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:22.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:22 smithi118 ceph-mon[49197]: osdmap e142: 8 total, 8 up, 8 in 2024-09-20T19:45:23.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 230200 9.23 MB/sec execute 493 sec latency 1865.892 ms 2024-09-20T19:45:23.750 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:23 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T19:45:23.751 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:23 smithi116 ceph-mon[63578]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T19:45:23.751 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:23 smithi116 ceph-mon[63578]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 355 op/s; 177/1638 objects degraded (10.806%) 2024-09-20T19:45:23.751 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:23 smithi116 ceph-mon[63578]: osdmap e143: 8 total, 8 up, 8 in 2024-09-20T19:45:23.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:23 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T19:45:23.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:23 smithi118 ceph-mon[49197]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T19:45:23.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:23 smithi118 ceph-mon[49197]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 355 op/s; 177/1638 objects degraded (10.806%) 2024-09-20T19:45:23.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:23 smithi118 ceph-mon[49197]: osdmap e143: 8 total, 8 up, 8 in 2024-09-20T19:45:24.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 230306 9.22 MB/sec execute 494 sec latency 2247.828 ms 2024-09-20T19:45:25.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 230339 9.20 MB/sec execute 495 sec latency 832.930 ms 2024-09-20T19:45:25.773 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:25 smithi116 ceph-mon[63578]: pgmap v246: 97 pgs: 1 remapped+peering, 1 active+recovering+degraded, 5 peering, 1 active+recovery_wait+degraded, 4 active+undersized, 12 active+undersized+degraded, 73 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.4 MiB/s wr, 323 op/s; 115/1557 objects degraded (7.386%); 82 KiB/s, 7 objects/s recovering 2024-09-20T19:45:25.774 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:25 smithi116 ceph-mon[63578]: osdmap e144: 8 total, 8 up, 8 in 2024-09-20T19:45:25.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:25 smithi118 ceph-mon[49197]: pgmap v246: 97 pgs: 1 remapped+peering, 1 active+recovering+degraded, 5 peering, 1 active+recovery_wait+degraded, 4 active+undersized, 12 active+undersized+degraded, 73 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.4 MiB/s wr, 323 op/s; 115/1557 objects degraded (7.386%); 82 KiB/s, 7 objects/s recovering 2024-09-20T19:45:25.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:25 smithi118 ceph-mon[49197]: osdmap e144: 8 total, 8 up, 8 in 2024-09-20T19:45:26.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 230525 9.19 MB/sec execute 496 sec latency 1053.957 ms 2024-09-20T19:45:26.587 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:26 smithi116 ceph-mon[63578]: osdmap e145: 8 total, 8 up, 8 in 2024-09-20T19:45:26.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:26 smithi118 ceph-mon[49197]: osdmap e145: 8 total, 8 up, 8 in 2024-09-20T19:45:27.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 230736 9.20 MB/sec execute 497 sec latency 86.506 ms 2024-09-20T19:45:27.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:27 smithi116 ceph-mon[63578]: pgmap v249: 97 pgs: 1 remapped+peering, 2 active+recovering+degraded, 5 peering, 1 active+recovery_wait+degraded, 88 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.5 MiB/s wr, 338 op/s; 21/1500 objects degraded (1.400%); 1.1 MiB/s, 37 objects/s recovering 2024-09-20T19:45:27.600 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:27 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 21/1500 objects degraded (1.400%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:27.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:27 smithi118 ceph-mon[49197]: pgmap v249: 97 pgs: 1 remapped+peering, 2 active+recovering+degraded, 5 peering, 1 active+recovery_wait+degraded, 88 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.5 MiB/s wr, 338 op/s; 21/1500 objects degraded (1.400%); 1.1 MiB/s, 37 objects/s recovering 2024-09-20T19:45:27.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:27 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 21/1500 objects degraded (1.400%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:28.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 231050 9.20 MB/sec execute 498 sec latency 60.553 ms 2024-09-20T19:45:28.630 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:28.631 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:28.631 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:28 smithi116 ceph-mon[63578]: pgmap v250: 97 pgs: 1 remapped+peering, 2 active+recovering+degraded, 1 active+recovery_wait+degraded, 93 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 MiB/s wr, 238 op/s; 21/1461 objects degraded (1.437%); 1.1 MiB/s, 32 objects/s recovering 2024-09-20T19:45:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:28 smithi118 ceph-mon[49197]: pgmap v250: 97 pgs: 1 remapped+peering, 2 active+recovering+degraded, 1 active+recovery_wait+degraded, 93 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 MiB/s wr, 238 op/s; 21/1461 objects degraded (1.437%); 1.1 MiB/s, 32 objects/s recovering 2024-09-20T19:45:29.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 231341 9.20 MB/sec execute 499 sec latency 74.973 ms 2024-09-20T19:45:30.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 231580 9.19 MB/sec execute 500 sec latency 83.013 ms 2024-09-20T19:45:30.984 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:30 smithi116 ceph-mon[63578]: pgmap v251: 97 pgs: 1 active+recovering+degraded, 96 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.8 MiB/s wr, 298 op/s; 939 KiB/s, 32 objects/s recovering 2024-09-20T19:45:31.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:30 smithi118 ceph-mon[49197]: pgmap v251: 97 pgs: 1 active+recovering+degraded, 96 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.8 MiB/s wr, 298 op/s; 939 KiB/s, 32 objects/s recovering 2024-09-20T19:45:31.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 231776 9.18 MB/sec execute 501 sec latency 55.868 ms 2024-09-20T19:45:32.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 231974 9.17 MB/sec execute 502 sec latency 80.360 ms 2024-09-20T19:45:32.423 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:45:33.071 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:32 smithi116 ceph-mon[63578]: pgmap v252: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.5 MiB/s wr, 261 op/s; 704 KiB/s, 23 objects/s recovering 2024-09-20T19:45:33.072 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:32 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-09-20T19:45:33.072 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:32 smithi116 ceph-mon[63578]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:33.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:32 smithi118 ceph-mon[49197]: pgmap v252: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.5 MiB/s wr, 261 op/s; 704 KiB/s, 23 objects/s recovering 2024-09-20T19:45:33.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:32 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-09-20T19:45:33.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:32 smithi118 ceph-mon[49197]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:33.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 232303 9.17 MB/sec execute 503 sec latency 86.361 ms 2024-09-20T19:45:33.407 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (7m) 94s ago 16m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (4m) 94s ago 16m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (4m) 16s ago 16m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (6m) 94s ago 16m 47.1M - dad864ee21e9 8a163a77164a 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (10m) 94s ago 10m 37.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (10m) 16s ago 10m 15.0M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (7m) 94s ago 18m 497M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (6m) 16s ago 16m 450M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (6m) 94s ago 18m 51.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (5m) 16s ago 16m 45.0M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (10m) 94s ago 10m 290M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (10m) 16s ago 10m 86.1M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:45:33.408 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (7m) 94s ago 16m 10.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 16s ago 16m 11.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (4m) 94s ago 15m 343M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (3m) 94s ago 14m 255M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (2m) 94s ago 14m 214M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (104s) 94s ago 14m 14.3M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (59s) 16s ago 13m 234M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (22s) 16s ago 13m 12.1M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (12m) 16s ago 12m 491M 2704M 17.2.0 e1d6a67b021e 70697719ce98 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (12m) 16s ago 12m 723M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:45:33.409 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (6m) 94s ago 15m 55.4M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 6 2024-09-20T19:45:33.976 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 10 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:45:33.977 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:45:34.101 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:33 smithi116 ceph-mon[63578]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:34.102 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:33 smithi116 ceph-mon[63578]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:34.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:33 smithi118 ceph-mon[49197]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:34.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:33 smithi118 ceph-mon[49197]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:34.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 232623 9.18 MB/sec execute 504 sec latency 100.627 ms 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "12/23 daemons upgraded", 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:45:34.445 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:45:35.037 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:45:35.142 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:34 smithi116 ceph-mon[63578]: pgmap v253: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 370 op/s; 606 KiB/s, 20 objects/s recovering 2024-09-20T19:45:35.142 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:34 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2195204549' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:35.143 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:34 smithi116 ceph-mon[63578]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:35.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:34 smithi118 ceph-mon[49197]: pgmap v253: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 370 op/s; 606 KiB/s, 20 objects/s recovering 2024-09-20T19:45:35.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:34 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2195204549' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:45:35.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:34 smithi118 ceph-mon[49197]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:45:35.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 232920 9.17 MB/sec execute 505 sec latency 122.983 ms 2024-09-20T19:45:36.156 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:35 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1511219621' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:45:36.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:35 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1511219621' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:45:36.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 233242 9.17 MB/sec execute 506 sec latency 72.444 ms 2024-09-20T19:45:36.929 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:36 smithi116 ceph-mon[63578]: pgmap v254: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.6 MiB/s wr, 346 op/s; 547 KiB/s, 16 objects/s recovering 2024-09-20T19:45:36.929 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T19:45:37.185 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:36 smithi118 ceph-mon[49197]: pgmap v254: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.6 MiB/s wr, 346 op/s; 547 KiB/s, 16 objects/s recovering 2024-09-20T19:45:37.186 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T19:45:37.309 INFO:teuthology.orchestra.run.smithi116.stdout: 5 233726 9.17 MB/sec execute 507 sec latency 34.339 ms 2024-09-20T19:45:37.995 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:37 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T19:45:37.996 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:37 smithi116 ceph-mon[63578]: Upgrade: osd.6 is safe to restart 2024-09-20T19:45:37.996 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:37 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:38.188 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:37 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T19:45:38.188 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:37 smithi118 ceph-mon[49197]: Upgrade: osd.6 is safe to restart 2024-09-20T19:45:38.188 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:37 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:38.318 INFO:teuthology.orchestra.run.smithi116.stdout: 5 234011 9.16 MB/sec execute 508 sec latency 47.051 ms 2024-09-20T19:45:39.318 INFO:teuthology.orchestra.run.smithi116.stdout: 5 234293 9.17 MB/sec execute 509 sec latency 83.180 ms 2024-09-20T19:45:39.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:39 smithi116 ceph-mon[63578]: pgmap v255: 97 pgs: 97 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 377 op/s; 108 KiB/s, 5 objects/s recovering 2024-09-20T19:45:39.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:39 smithi116 ceph-mon[63578]: Upgrade: Updating osd.6 2024-09-20T19:45:39.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:39.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T19:45:39.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:45:39.510 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:39 smithi116 ceph-mon[63578]: Deploying daemon osd.6 on smithi118 2024-09-20T19:45:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:39 smithi118 ceph-mon[49197]: pgmap v255: 97 pgs: 97 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 377 op/s; 108 KiB/s, 5 objects/s recovering 2024-09-20T19:45:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:39 smithi118 ceph-mon[49197]: Upgrade: Updating osd.6 2024-09-20T19:45:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T19:45:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:45:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:39 smithi118 ceph-mon[49197]: Deploying daemon osd.6 on smithi118 2024-09-20T19:45:40.318 INFO:teuthology.orchestra.run.smithi116.stdout: 5 234661 9.17 MB/sec execute 510 sec latency 58.071 ms 2024-09-20T19:45:41.318 INFO:teuthology.orchestra.run.smithi116.stdout: 5 234994 9.17 MB/sec execute 511 sec latency 63.291 ms 2024-09-20T19:45:41.549 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:41 smithi116 ceph-mon[63578]: pgmap v256: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.2 MiB/s wr, 399 op/s; 14 KiB/s, 3 objects/s recovering 2024-09-20T19:45:41.604 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:41 smithi118 ceph-mon[49197]: pgmap v256: 97 pgs: 97 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.2 MiB/s wr, 399 op/s; 14 KiB/s, 3 objects/s recovering 2024-09-20T19:45:42.318 INFO:teuthology.orchestra.run.smithi116.stdout: 5 235315 9.17 MB/sec execute 512 sec latency 91.390 ms 2024-09-20T19:45:42.927 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:42 smithi118 systemd[1]: Stopping Ceph osd.6 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:45:43.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:42 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[35895]: 2024-09-20T19:45:42.924+0000 7f60205c2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:45:43.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:42 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[35895]: 2024-09-20T19:45:42.924+0000 7f60205c2700 -1 osd.6 145 *** Got signal Terminated *** 2024-09-20T19:45:43.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:42 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[35895]: 2024-09-20T19:45:42.924+0000 7f60205c2700 -1 osd.6 145 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:45:43.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 235523 9.16 MB/sec execute 513 sec latency 69.323 ms 2024-09-20T19:45:43.496 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:43 smithi116 ceph-mon[63578]: pgmap v257: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.9 MiB/s wr, 413 op/s 2024-09-20T19:45:43.497 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:43.497 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:43.497 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:43 smithi116 ceph-mon[63578]: osd.6 marked itself down and dead 2024-09-20T19:45:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:43 smithi118 ceph-mon[49197]: pgmap v257: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.9 MiB/s wr, 413 op/s 2024-09-20T19:45:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:43.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:43 smithi118 ceph-mon[49197]: osd.6 marked itself down and dead 2024-09-20T19:45:44.318 INFO:teuthology.orchestra.run.smithi116.stdout: 5 235610 9.15 MB/sec execute 514 sec latency 776.856 ms 2024-09-20T19:45:44.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:44 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:45:44.451 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:44 smithi118 ceph-mon[49197]: osdmap e146: 8 total, 7 up, 8 in 2024-09-20T19:45:44.451 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:44 smithi118 podman[56731]: 2024-09-20 19:45:44.189121435 +0000 UTC m=+1.386537641 container died 70697719ce9860a5a1e5d26f766dcd3adf0284467216a4d437c46d81e8ab387c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, release=754, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container) 2024-09-20T19:45:44.665 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:44 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:45:44.666 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:44 smithi116 ceph-mon[63578]: osdmap e146: 8 total, 7 up, 8 in 2024-09-20T19:45:44.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:44 smithi118 podman[56731]: 2024-09-20 19:45:44.549352322 +0000 UTC m=+1.746768528 container cleanup 70697719ce9860a5a1e5d26f766dcd3adf0284467216a4d437c46d81e8ab387c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, vcs-type=git, name=centos-stream, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.expose-services=, RELEASE=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8) 2024-09-20T19:45:44.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:44 smithi118 bash[56731]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6 2024-09-20T19:45:45.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 235779 9.14 MB/sec execute 515 sec latency 1075.900 ms 2024-09-20T19:45:45.422 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:45 smithi118 podman[56746]: 2024-09-20 19:45:45.068022546 +0000 UTC m=+0.877304835 container remove 70697719ce9860a5a1e5d26f766dcd3adf0284467216a4d437c46d81e8ab387c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_CLEAN=True, distribution-scope=public, RELEASE=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, release=754, ceph=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, name=centos-stream) 2024-09-20T19:45:45.440 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:45 smithi116 ceph-mon[63578]: pgmap v259: 97 pgs: 11 peering, 6 stale+active+clean, 80 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-20T19:45:45.440 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:45 smithi116 ceph-mon[63578]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T19:45:45.441 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:45 smithi116 ceph-mon[63578]: osdmap e147: 8 total, 7 up, 8 in 2024-09-20T19:45:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:45 smithi118 ceph-mon[49197]: pgmap v259: 97 pgs: 11 peering, 6 stale+active+clean, 80 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-20T19:45:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:45 smithi118 ceph-mon[49197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T19:45:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:45 smithi118 ceph-mon[49197]: osdmap e147: 8 total, 7 up, 8 in 2024-09-20T19:45:45.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:45 smithi118 podman[56833]: 2024-09-20 19:45:45.350706299 +0000 UTC m=+0.017440762 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:45:45.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:45 smithi118 podman[56833]: 2024-09-20 19:45:45.4775233 +0000 UTC m=+0.144257773 container create 12d9dc58e9b5bc3140d366f4ed551d3c76766f32193e21138e8efc7b96031d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-20T19:45:46.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:45 smithi118 podman[56833]: 2024-09-20 19:45:45.747378954 +0000 UTC m=+0.414113435 container init 12d9dc58e9b5bc3140d366f4ed551d3c76766f32193e21138e8efc7b96031d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-20T19:45:46.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:45 smithi118 podman[56833]: 2024-09-20 19:45:45.752065483 +0000 UTC m=+0.418799953 container start 12d9dc58e9b5bc3140d366f4ed551d3c76766f32193e21138e8efc7b96031d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:45:46.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:45 smithi118 podman[56833]: 2024-09-20 19:45:45.847526429 +0000 UTC m=+0.514260894 container attach 12d9dc58e9b5bc3140d366f4ed551d3c76766f32193e21138e8efc7b96031d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:45:46.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:45 smithi118 podman[56833]: 2024-09-20 19:45:45.970397766 +0000 UTC m=+0.637132243 container died 12d9dc58e9b5bc3140d366f4ed551d3c76766f32193e21138e8efc7b96031d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:45:46.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 236150 9.15 MB/sec execute 516 sec latency 63.852 ms 2024-09-20T19:45:46.706 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:46 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 67/1629 objects degraded (4.113%), 8 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:46.946 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:46 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 67/1629 objects degraded (4.113%), 8 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:47.210 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:46 smithi118 podman[56856]: 2024-09-20 19:45:46.945519068 +0000 UTC m=+0.966789458 container remove 12d9dc58e9b5bc3140d366f4ed551d3c76766f32193e21138e8efc7b96031d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-20T19:45:47.210 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:46 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-20T19:45:47.210 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:46 smithi118 systemd[1]: Stopped Ceph osd.6 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:45:47.210 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:46 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.6.service: Consumed 4min 4.634s CPU time. 2024-09-20T19:45:47.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 236441 9.14 MB/sec execute 517 sec latency 47.647 ms 2024-09-20T19:45:47.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:47 smithi118 systemd[1]: Starting Ceph osd.6 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:45:47.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:47 smithi118 podman[56955]: 2024-09-20 19:45:47.327910261 +0000 UTC m=+0.017274502 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:45:47.519 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:47 smithi116 ceph-mon[63578]: pgmap v261: 97 pgs: 3 active+undersized, 11 peering, 3 stale+active+clean, 8 active+undersized+degraded, 72 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 349 op/s; 67/1629 objects degraded (4.113%); 8.0 KiB/s, 0 objects/s recovering 2024-09-20T19:45:47.919 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:47 smithi118 ceph-mon[49197]: pgmap v261: 97 pgs: 3 active+undersized, 11 peering, 3 stale+active+clean, 8 active+undersized+degraded, 72 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 349 op/s; 67/1629 objects degraded (4.113%); 8.0 KiB/s, 0 objects/s recovering 2024-09-20T19:45:47.919 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:47 smithi118 podman[56955]: 2024-09-20 19:45:47.640739166 +0000 UTC m=+0.330103438 container create b1209691bec35c869482b9b1561f38b9a5edcd651835a9a1d4548535540d98da (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-20T19:45:48.205 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:48 smithi118 podman[56955]: 2024-09-20 19:45:48.054877349 +0000 UTC m=+0.744241578 container init b1209691bec35c869482b9b1561f38b9a5edcd651835a9a1d4548535540d98da (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-20T19:45:48.205 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:48 smithi118 podman[56955]: 2024-09-20 19:45:48.058454619 +0000 UTC m=+0.747818856 container start b1209691bec35c869482b9b1561f38b9a5edcd651835a9a1d4548535540d98da (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:45:48.205 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:48 smithi118 podman[56955]: 2024-09-20 19:45:48.156481829 +0000 UTC m=+0.845846076 container attach b1209691bec35c869482b9b1561f38b9a5edcd651835a9a1d4548535540d98da (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:45:48.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 236709 9.14 MB/sec execute 518 sec latency 52.688 ms 2024-09-20T19:45:48.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:48 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:48.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:48 smithi118 bash[56955]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:48.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:48 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:48.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:48 smithi118 bash[56955]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:45:49.292 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T19:45:49.292 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 bash[56955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T19:45:49.292 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T19:45:49.293 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 bash[56955]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-20T19:45:49.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 237001 9.13 MB/sec execute 519 sec latency 83.368 ms 2024-09-20T19:45:49.567 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:49 smithi116 ceph-mon[63578]: pgmap v262: 97 pgs: 5 active+undersized, 11 peering, 1 stale+active+clean, 14 active+undersized+degraded, 66 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 446 op/s; 118/1647 objects degraded (7.165%); 8.0 KiB/s, 0 objects/s recovering 2024-09-20T19:45:49.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-mon[49197]: pgmap v262: 97 pgs: 5 active+undersized, 11 peering, 1 stale+active+clean, 14 active+undersized+degraded, 66 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 446 op/s; 118/1647 objects degraded (7.165%); 8.0 KiB/s, 0 objects/s recovering 2024-09-20T19:45:49.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T19:45:49.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 bash[56955]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-20T19:45:49.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T19:45:49.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 bash[56955]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T19:45:49.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-20T19:45:49.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 bash[56955]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-20T19:45:49.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T19:45:49.716 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 bash[56955]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T19:45:49.716 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate[56970]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-20T19:45:49.716 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 bash[56955]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-20T19:45:49.716 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:49 smithi118 podman[56955]: 2024-09-20 19:45:49.326335632 +0000 UTC m=+2.015699864 container died b1209691bec35c869482b9b1561f38b9a5edcd651835a9a1d4548535540d98da (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-20T19:45:50.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 237321 9.12 MB/sec execute 520 sec latency 100.082 ms 2024-09-20T19:45:50.614 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:50 smithi116 ceph-mon[63578]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T19:45:50.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:50 smithi118 ceph-mon[49197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T19:45:51.210 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:50 smithi118 podman[57096]: 2024-09-20 19:45:50.94175773 +0000 UTC m=+1.609348176 container remove b1209691bec35c869482b9b1561f38b9a5edcd651835a9a1d4548535540d98da (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-activate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-20T19:45:51.210 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:51 smithi118 podman[57153]: 2024-09-20 19:45:51.079869839 +0000 UTC m=+0.020742541 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:45:51.211 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:51 smithi118 podman[57153]: 2024-09-20 19:45:51.20959822 +0000 UTC m=+0.150470908 container create 14b930edda7f29b20eb0c28cbc0996b3dcb8054b79916a126a345561c019f214 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-09-20T19:45:51.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 237600 9.12 MB/sec execute 521 sec latency 43.503 ms 2024-09-20T19:45:51.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:51 smithi116 ceph-mon[63578]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 377 op/s; 192/1662 objects degraded (11.552%); 8.0 KiB/s, 0 objects/s recovering 2024-09-20T19:45:51.786 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:51 smithi118 ceph-mon[49197]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 377 op/s; 192/1662 objects degraded (11.552%); 8.0 KiB/s, 0 objects/s recovering 2024-09-20T19:45:51.786 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:51 smithi118 podman[57153]: 2024-09-20 19:45:51.754495943 +0000 UTC m=+0.695368631 container init 14b930edda7f29b20eb0c28cbc0996b3dcb8054b79916a126a345561c019f214 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T19:45:51.786 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:51 smithi118 podman[57153]: 2024-09-20 19:45:51.758785989 +0000 UTC m=+0.699658685 container start 14b930edda7f29b20eb0c28cbc0996b3dcb8054b79916a126a345561c019f214 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:45:52.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:51 smithi118 ceph-osd[57170]: -- 172.21.15.118:0/1641255279 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a825a181e0 con 0x55a825a14000 2024-09-20T19:45:52.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:51 smithi118 bash[57153]: 14b930edda7f29b20eb0c28cbc0996b3dcb8054b79916a126a345561c019f214 2024-09-20T19:45:52.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:51 smithi118 systemd[1]: Started Ceph osd.6 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:45:52.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 237805 9.11 MB/sec execute 522 sec latency 47.550 ms 2024-09-20T19:45:52.975 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:52 smithi116 ceph-mon[63578]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.6 MiB/s wr, 435 op/s; 185/1653 objects degraded (11.192%); 7.8 KiB/s, 0 objects/s recovering 2024-09-20T19:45:52.976 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:52 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:52.976 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:52 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:52.976 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:52 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:45:52.976 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:52 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 192/1662 objects degraded (11.552%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:53.215 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:53 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:45:53.139+0000 7fc5703a5640 -1 Falling back to public interface 2024-09-20T19:45:53.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:52 smithi118 ceph-mon[49197]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.6 MiB/s wr, 435 op/s; 185/1653 objects degraded (11.192%); 7.8 KiB/s, 0 objects/s recovering 2024-09-20T19:45:53.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:52 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:53.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:52 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:53.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:52 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:45:53.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:52 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 192/1662 objects degraded (11.552%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:53.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 238070 9.12 MB/sec execute 523 sec latency 86.467 ms 2024-09-20T19:45:54.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 238375 9.12 MB/sec execute 524 sec latency 76.781 ms 2024-09-20T19:45:55.085 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:54 smithi116 ceph-mon[63578]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 182/1599 objects degraded (11.382%); 6.4 KiB/s, 0 objects/s recovering 2024-09-20T19:45:55.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:54 smithi118 ceph-mon[49197]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 405 op/s; 182/1599 objects degraded (11.382%); 6.4 KiB/s, 0 objects/s recovering 2024-09-20T19:45:55.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 238687 9.11 MB/sec execute 525 sec latency 65.092 ms 2024-09-20T19:45:56.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 238940 9.11 MB/sec execute 526 sec latency 51.665 ms 2024-09-20T19:45:57.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 239170 9.10 MB/sec execute 527 sec latency 55.265 ms 2024-09-20T19:45:57.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:57 smithi116 ceph-mon[63578]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 475 op/s; 180/1599 objects degraded (11.257%); 5.7 KiB/s, 0 objects/s recovering 2024-09-20T19:45:57.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:57.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:57.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:57.400 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:57.491 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:57 smithi118 ceph-mon[49197]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 475 op/s; 180/1599 objects degraded (11.257%); 5.7 KiB/s, 0 objects/s recovering 2024-09-20T19:45:57.491 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:57.491 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:57.491 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:57.491 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:58.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 239451 9.10 MB/sec execute 528 sec latency 61.966 ms 2024-09-20T19:45:58.412 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:58 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 180/1599 objects degraded (11.257%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:58.412 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:58.412 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:58.652 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:58 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 180/1599 objects degraded (11.257%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T19:45:58.652 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:45:58.652 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:45:59.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 239792 9.11 MB/sec execute 529 sec latency 71.917 ms 2024-09-20T19:45:59.373 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:45:59 smithi118 ceph-mon[49197]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.0 MiB/s wr, 410 op/s; 182/1629 objects degraded (11.172%) 2024-09-20T19:45:59.373 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:45:59 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:45:59.295+0000 7fc5703a5640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-20T19:45:59.436 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:45:59 smithi116 ceph-mon[63578]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.0 MiB/s wr, 410 op/s; 182/1629 objects degraded (11.172%) 2024-09-20T19:46:00.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 240061 9.10 MB/sec execute 530 sec latency 75.935 ms 2024-09-20T19:46:00.590 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:46:00 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:46:00.394+0000 7fc5703a5640 -1 osd.6 145 log_to_monitors true 2024-09-20T19:46:01.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 240325 9.09 MB/sec execute 531 sec latency 58.301 ms 2024-09-20T19:46:01.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 338 op/s; 191/1677 objects degraded (11.389%) 2024-09-20T19:46:01.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='osd.6 [v2:172.21.15.118:6816/2823124196,v1:172.21.15.118:6817/2823124196]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T19:46:01.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T19:46:01.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:01.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:46:01.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 338 op/s; 191/1677 objects degraded (11.389%) 2024-09-20T19:46:01.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='osd.6 [v2:172.21.15.118:6816/2823124196,v1:172.21.15.118:6817/2823124196]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T19:46:01.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T19:46:01.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.481 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:01.482 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:46:01.482 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.482 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:01 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:01.965 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:46:01 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:46:01.564+0000 7fc564147640 -1 osd.6 145 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:46:02.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 240552 9.09 MB/sec execute 532 sec latency 89.208 ms 2024-09-20T19:46:02.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T19:46:02.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='osd.6 [v2:172.21.15.118:6816/2823124196,v1:172.21.15.118:6817/2823124196]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:02.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: osdmap e148: 8 total, 7 up, 8 in 2024-09-20T19:46:02.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:02.371 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:02.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:02.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:02.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:02.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T19:46:02.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T19:46:02.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T19:46:02.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:02 smithi116 ceph-mon[63578]: from='osd.6 ' entity='osd.6' 2024-09-20T19:46:02.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T19:46:02.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='osd.6 [v2:172.21.15.118:6816/2823124196,v1:172.21.15.118:6817/2823124196]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:02.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: osdmap e148: 8 total, 7 up, 8 in 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T19:46:02.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:02 smithi118 ceph-mon[49197]: from='osd.6 ' entity='osd.6' 2024-09-20T19:46:03.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 240698 9.07 MB/sec execute 533 sec latency 750.034 ms 2024-09-20T19:46:03.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:03 smithi116 ceph-mon[63578]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 344 op/s; 190/1674 objects degraded (11.350%) 2024-09-20T19:46:03.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:03 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 191/1677 objects degraded (11.389%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:03.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:03 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:46:03.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:03 smithi116 ceph-mon[63578]: osd.6 [v2:172.21.15.118:6816/2823124196,v1:172.21.15.118:6817/2823124196] boot 2024-09-20T19:46:03.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:03 smithi116 ceph-mon[63578]: osdmap e149: 8 total, 8 up, 8 in 2024-09-20T19:46:03.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:03 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T19:46:03.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:03 smithi118 ceph-mon[49197]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 344 op/s; 190/1674 objects degraded (11.350%) 2024-09-20T19:46:03.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:03 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 191/1677 objects degraded (11.389%), 25 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:03.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:03 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:46:03.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:03 smithi118 ceph-mon[49197]: osd.6 [v2:172.21.15.118:6816/2823124196,v1:172.21.15.118:6817/2823124196] boot 2024-09-20T19:46:03.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:03 smithi118 ceph-mon[49197]: osdmap e149: 8 total, 8 up, 8 in 2024-09-20T19:46:03.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:03 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T19:46:04.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 240969 9.07 MB/sec execute 534 sec latency 1104.175 ms 2024-09-20T19:46:04.648 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:04 smithi116 ceph-mon[63578]: osdmap e150: 8 total, 8 up, 8 in 2024-09-20T19:46:04.649 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:04 smithi116 ceph-mon[63578]: pgmap v273: 97 pgs: 1 remapped+peering, 1 activating+degraded, 4 activating, 4 peering, 5 active+undersized, 19 active+undersized+degraded, 63 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.8 MiB/s wr, 298 op/s; 146/1614 objects degraded (9.046%) 2024-09-20T19:46:04.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:04 smithi118 ceph-mon[49197]: osdmap e150: 8 total, 8 up, 8 in 2024-09-20T19:46:04.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:04 smithi118 ceph-mon[49197]: pgmap v273: 97 pgs: 1 remapped+peering, 1 activating+degraded, 4 activating, 4 peering, 5 active+undersized, 19 active+undersized+degraded, 63 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.8 MiB/s wr, 298 op/s; 146/1614 objects degraded (9.046%) 2024-09-20T19:46:05.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 241285 9.07 MB/sec execute 535 sec latency 1653.002 ms 2024-09-20T19:46:05.608 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:46:05.678 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:05 smithi116 ceph-mon[63578]: osdmap e151: 8 total, 8 up, 8 in 2024-09-20T19:46:05.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:05 smithi118 ceph-mon[49197]: osdmap e151: 8 total, 8 up, 8 in 2024-09-20T19:46:06.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 241594 9.07 MB/sec execute 536 sec latency 530.719 ms 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (8m) 2m ago 17m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (5m) 2m ago 17m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (5m) 10s ago 16m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (7m) 2m ago 17m 47.1M - dad864ee21e9 8a163a77164a 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (11m) 2m ago 11m 37.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (11m) 10s ago 11m 15.2M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (7m) 2m ago 18m 497M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:46:06.505 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (6m) 10s ago 16m 450M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (6m) 2m ago 18m 51.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (6m) 10s ago 16m 46.0M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (11m) 2m ago 11m 290M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (11m) 10s ago 11m 86.1M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (8m) 2m ago 17m 10.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (7m) 10s ago 16m 10.9M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (4m) 2m ago 15m 343M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (3m) 2m ago 15m 255M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (3m) 2m ago 14m 214M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (2m) 2m ago 14m 14.3M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (92s) 10s ago 14m 271M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (55s) 10s ago 13m 253M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (15s) 10s ago 13m 12.0M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 14b930edda7f 2024-09-20T19:46:06.506 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (13m) 10s ago 12m 753M 2704M 17.2.0 e1d6a67b021e b5d1edee7952 2024-09-20T19:46:06.507 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (7m) 2m ago 16m 55.4M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:46:06.760 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:06 smithi116 ceph-mon[63578]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:06.761 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:06 smithi116 ceph-mon[63578]: osdmap e152: 8 total, 8 up, 8 in 2024-09-20T19:46:06.761 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:06 smithi116 ceph-mon[63578]: pgmap v276: 97 pgs: 1 remapped+peering, 1 activating+degraded, 4 activating, 4 peering, 3 active+undersized, 13 active+undersized+degraded, 71 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 159 op/s; 94/1596 objects degraded (5.890%); 163 KiB/s, 57 keys/s, 13 objects/s recovering 2024-09-20T19:46:06.761 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:06 smithi116 ceph-mon[63578]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:06.761 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:06 smithi116 ceph-mon[63578]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:06.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:06 smithi118 ceph-mon[49197]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:06.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:06 smithi118 ceph-mon[49197]: osdmap e152: 8 total, 8 up, 8 in 2024-09-20T19:46:06.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:06 smithi118 ceph-mon[49197]: pgmap v276: 97 pgs: 1 remapped+peering, 1 activating+degraded, 4 activating, 4 peering, 3 active+undersized, 13 active+undersized+degraded, 71 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.8 MiB/s wr, 159 op/s; 94/1596 objects degraded (5.890%); 163 KiB/s, 57 keys/s, 13 objects/s recovering 2024-09-20T19:46:06.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:06 smithi118 ceph-mon[49197]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:06.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:06 smithi118 ceph-mon[49197]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 7 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:46:07.127 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:46:07.128 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:07.128 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:46:07.128 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-20T19:46:07.128 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 11 2024-09-20T19:46:07.128 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:46:07.128 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:46:07.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 241811 9.07 MB/sec execute 537 sec latency 820.612 ms 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: "crash" 2024-09-20T19:46:07.599 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:46:07.600 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "13/23 daemons upgraded", 2024-09-20T19:46:07.600 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T19:46:07.600 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:46:07.600 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:46:07.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:07 smithi116 ceph-mon[63578]: osdmap e153: 8 total, 8 up, 8 in 2024-09-20T19:46:07.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:07 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2174071548' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:07.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:07 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 94/1596 objects degraded (5.890%), 14 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:07.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:07 smithi118 ceph-mon[49197]: osdmap e153: 8 total, 8 up, 8 in 2024-09-20T19:46:07.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:07 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2174071548' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:07.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:07 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 94/1596 objects degraded (5.890%), 14 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); Degraded data redundancy: 94/1596 objects degraded (5.890%), 14 pgs degraded; 5 mgr modules have recently crashed 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 94/1596 objects degraded (5.890%), 14 pgs degraded 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-20T19:46:08.318 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.3 is activating+degraded, acting [4,0,6] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.a is active+undersized+degraded, acting [3,1] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:46:08.319 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:46:08.320 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:46:08.320 INFO:teuthology.orchestra.run.smithi116.stdout: 5 242097 9.07 MB/sec execute 538 sec latency 58.480 ms 2024-09-20T19:46:08.783 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:08 smithi116 ceph-mon[63578]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:08.783 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:08 smithi116 ceph-mon[63578]: pgmap v278: 97 pgs: 1 remapped+peering, 1 activating+degraded, 4 activating, 5 peering, 86 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 654 op/s; 17/1590 objects degraded (1.069%); 1.0 MiB/s, 54 keys/s, 32 objects/s recovering 2024-09-20T19:46:08.784 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:08 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/4258032658' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:46:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:08 smithi118 ceph-mon[49197]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:08 smithi118 ceph-mon[49197]: pgmap v278: 97 pgs: 1 remapped+peering, 1 activating+degraded, 4 activating, 5 peering, 86 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 654 op/s; 17/1590 objects degraded (1.069%); 1.0 MiB/s, 54 keys/s, 32 objects/s recovering 2024-09-20T19:46:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:08 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/4258032658' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:46:09.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 242443 9.07 MB/sec execute 539 sec latency 74.718 ms 2024-09-20T19:46:10.059 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:09 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/1590 objects degraded (1.069%), 1 pg degraded) 2024-09-20T19:46:10.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:09 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/1590 objects degraded (1.069%), 1 pg degraded) 2024-09-20T19:46:10.319 INFO:teuthology.orchestra.run.smithi116.stdout: 5 242667 9.06 MB/sec execute 540 sec latency 59.656 ms 2024-09-20T19:46:11.104 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:10 smithi116 ceph-mon[63578]: pgmap v279: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 494 op/s; 783 KiB/s, 38 keys/s, 35 objects/s recovering 2024-09-20T19:46:11.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:10 smithi118 ceph-mon[49197]: pgmap v279: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 494 op/s; 783 KiB/s, 38 keys/s, 35 objects/s recovering 2024-09-20T19:46:11.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 242893 9.06 MB/sec execute 541 sec latency 52.595 ms 2024-09-20T19:46:12.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 243198 9.06 MB/sec execute 542 sec latency 50.774 ms 2024-09-20T19:46:12.935 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:12 smithi116 ceph-mon[63578]: pgmap v280: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.1 MiB/s wr, 452 op/s; 1.2 MiB/s, 31 keys/s, 29 objects/s recovering 2024-09-20T19:46:12.936 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:12.936 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:13.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:12 smithi118 ceph-mon[49197]: pgmap v280: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.1 MiB/s wr, 452 op/s; 1.2 MiB/s, 31 keys/s, 29 objects/s recovering 2024-09-20T19:46:13.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:13.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:13.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 243434 9.05 MB/sec execute 543 sec latency 80.687 ms 2024-09-20T19:46:14.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 243749 9.05 MB/sec execute 544 sec latency 104.918 ms 2024-09-20T19:46:15.003 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:14 smithi116 ceph-mon[63578]: pgmap v281: 97 pgs: 97 active+clean; 344 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 558 op/s; 992 KiB/s, 19 objects/s recovering 2024-09-20T19:46:15.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:14 smithi118 ceph-mon[49197]: pgmap v281: 97 pgs: 97 active+clean; 344 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 558 op/s; 992 KiB/s, 19 objects/s recovering 2024-09-20T19:46:15.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 244051 9.04 MB/sec execute 545 sec latency 64.795 ms 2024-09-20T19:46:16.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 244451 9.04 MB/sec execute 546 sec latency 39.654 ms 2024-09-20T19:46:17.037 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:16 smithi116 ceph-mon[63578]: pgmap v282: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 476 op/s; 814 KiB/s, 15 objects/s recovering 2024-09-20T19:46:17.037 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T19:46:17.037 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:16 smithi116 ceph-mon[63578]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T19:46:17.037 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:16 smithi116 ceph-mon[63578]: Upgrade: osd.7 is safe to restart 2024-09-20T19:46:17.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:16 smithi118 ceph-mon[49197]: pgmap v282: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 476 op/s; 814 KiB/s, 15 objects/s recovering 2024-09-20T19:46:17.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T19:46:17.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:16 smithi118 ceph-mon[49197]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T19:46:17.171 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:16 smithi118 ceph-mon[49197]: Upgrade: osd.7 is safe to restart 2024-09-20T19:46:17.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 244815 9.05 MB/sec execute 547 sec latency 66.057 ms 2024-09-20T19:46:18.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 245054 9.04 MB/sec execute 548 sec latency 76.202 ms 2024-09-20T19:46:18.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:18.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:18 smithi118 ceph-mon[49197]: pgmap v283: 97 pgs: 97 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.6 MiB/s wr, 500 op/s; 676 KiB/s, 10 objects/s recovering 2024-09-20T19:46:18.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:18 smithi118 ceph-mon[49197]: Upgrade: Updating osd.7 2024-09-20T19:46:18.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:18.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T19:46:18.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:18.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:18 smithi118 ceph-mon[49197]: Deploying daemon osd.7 on smithi118 2024-09-20T19:46:18.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:18.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:18 smithi116 ceph-mon[63578]: pgmap v283: 97 pgs: 97 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.6 MiB/s wr, 500 op/s; 676 KiB/s, 10 objects/s recovering 2024-09-20T19:46:18.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:18 smithi116 ceph-mon[63578]: Upgrade: Updating osd.7 2024-09-20T19:46:18.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:18.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T19:46:18.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:18.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:18 smithi116 ceph-mon[63578]: Deploying daemon osd.7 on smithi118 2024-09-20T19:46:19.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 245359 9.04 MB/sec execute 549 sec latency 51.164 ms 2024-09-20T19:46:20.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 245761 9.05 MB/sec execute 550 sec latency 64.292 ms 2024-09-20T19:46:20.961 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:20 smithi116 ceph-mon[63578]: pgmap v284: 97 pgs: 97 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.2 MiB/s wr, 390 op/s; 363 KiB/s, 6 objects/s recovering 2024-09-20T19:46:21.106 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:20 smithi118 ceph-mon[49197]: pgmap v284: 97 pgs: 97 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.2 MiB/s wr, 390 op/s; 363 KiB/s, 6 objects/s recovering 2024-09-20T19:46:21.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 246053 9.05 MB/sec execute 551 sec latency 64.968 ms 2024-09-20T19:46:22.127 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:21 smithi118 systemd[1]: Stopping Ceph osd.7 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:46:22.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 246374 9.05 MB/sec execute 552 sec latency 39.750 ms 2024-09-20T19:46:22.465 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[38873]: 2024-09-20T19:46:22.125+0000 7f6c5214f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:46:22.465 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[38873]: 2024-09-20T19:46:22.125+0000 7f6c5214f700 -1 osd.7 153 *** Got signal Terminated *** 2024-09-20T19:46:22.465 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[38873]: 2024-09-20T19:46:22.125+0000 7f6c5214f700 -1 osd.7 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:46:23.008 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:22 smithi116 ceph-mon[63578]: pgmap v285: 97 pgs: 97 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 447 op/s; 341 KiB/s, 0 objects/s recovering 2024-09-20T19:46:23.008 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:22 smithi116 ceph-mon[63578]: osd.7 marked itself down and dead 2024-09-20T19:46:23.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:22 smithi118 ceph-mon[49197]: pgmap v285: 97 pgs: 97 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 447 op/s; 341 KiB/s, 0 objects/s recovering 2024-09-20T19:46:23.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:22 smithi118 ceph-mon[49197]: osd.7 marked itself down and dead 2024-09-20T19:46:23.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 246626 9.04 MB/sec execute 553 sec latency 439.973 ms 2024-09-20T19:46:23.857 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:23 smithi118 podman[58724]: 2024-09-20 19:46:23.581084249 +0000 UTC m=+1.575575443 container died b5d1edee7952752b5c1a1b2e9a8167956f9da0ed4f1fcaf3a65ab319cd278b38 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, version=8, release=754, GIT_BRANCH=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.component=centos-stream-container) 2024-09-20T19:46:24.107 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:23 smithi116 ceph-mon[63578]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:46:24.107 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:23 smithi116 ceph-mon[63578]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-20T19:46:24.108 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:23 smithi116 ceph-mon[63578]: osdmap e154: 8 total, 7 up, 8 in 2024-09-20T19:46:24.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:23 smithi118 ceph-mon[49197]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T19:46:24.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:23 smithi118 ceph-mon[49197]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-20T19:46:24.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:23 smithi118 ceph-mon[49197]: osdmap e154: 8 total, 7 up, 8 in 2024-09-20T19:46:24.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:23 smithi118 podman[58724]: 2024-09-20 19:46:23.967756185 +0000 UTC m=+1.962247374 container cleanup b5d1edee7952752b5c1a1b2e9a8167956f9da0ed4f1fcaf3a65ab319cd278b38 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., version=8, GIT_BRANCH=HEAD, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, RELEASE=HEAD, release=754, io.openshift.expose-services=) 2024-09-20T19:46:24.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:23 smithi118 bash[58724]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7 2024-09-20T19:46:24.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 246753 9.03 MB/sec execute 554 sec latency 1055.708 ms 2024-09-20T19:46:24.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:24 smithi118 podman[58740]: 2024-09-20 19:46:24.410909412 +0000 UTC m=+0.825827708 container remove b5d1edee7952752b5c1a1b2e9a8167956f9da0ed4f1fcaf3a65ab319cd278b38 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, distribution-scope=public, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.expose-services=, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-20T19:46:24.887 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:24 smithi116 ceph-mon[63578]: pgmap v287: 97 pgs: 14 peering, 83 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 459 op/s 2024-09-20T19:46:24.888 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:24 smithi116 ceph-mon[63578]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T19:46:24.888 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:24 smithi116 ceph-mon[63578]: osdmap e155: 8 total, 7 up, 8 in 2024-09-20T19:46:24.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:24 smithi118 ceph-mon[49197]: pgmap v287: 97 pgs: 14 peering, 83 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 459 op/s 2024-09-20T19:46:24.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:24 smithi118 ceph-mon[49197]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T19:46:24.986 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:24 smithi118 ceph-mon[49197]: osdmap e155: 8 total, 7 up, 8 in 2024-09-20T19:46:24.986 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:24 smithi118 podman[58830]: 2024-09-20 19:46:24.634929541 +0000 UTC m=+0.015678755 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:46:24.986 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:24 smithi118 podman[58830]: 2024-09-20 19:46:24.753556936 +0000 UTC m=+0.134306152 container create 77244cd98abdeb19efb54f5f00f26f25ee89c2376b048c05e711b855eadf30a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS) 2024-09-20T19:46:24.986 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:24 smithi118 podman[58830]: 2024-09-20 19:46:24.981462219 +0000 UTC m=+0.362211433 container init 77244cd98abdeb19efb54f5f00f26f25ee89c2376b048c05e711b855eadf30a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-20T19:46:24.987 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:24 smithi118 podman[58830]: 2024-09-20 19:46:24.984815281 +0000 UTC m=+0.365564495 container start 77244cd98abdeb19efb54f5f00f26f25ee89c2376b048c05e711b855eadf30a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-20T19:46:25.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 247048 9.04 MB/sec execute 555 sec latency 59.310 ms 2024-09-20T19:46:25.465 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:25 smithi118 podman[58830]: 2024-09-20 19:46:25.073100677 +0000 UTC m=+0.453849900 container attach 77244cd98abdeb19efb54f5f00f26f25ee89c2376b048c05e711b855eadf30a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True) 2024-09-20T19:46:25.465 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:25 smithi118 podman[58830]: 2024-09-20 19:46:25.1816142 +0000 UTC m=+0.562363416 container died 77244cd98abdeb19efb54f5f00f26f25ee89c2376b048c05e711b855eadf30a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-20T19:46:25.959 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:25 smithi116 ceph-mon[63578]: Health check failed: Degraded data redundancy: 41/1695 objects degraded (2.419%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:25 smithi118 ceph-mon[49197]: Health check failed: Degraded data redundancy: 41/1695 objects degraded (2.419%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:26.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:25 smithi118 podman[58853]: 2024-09-20 19:46:25.777756291 +0000 UTC m=+0.590351401 container remove 77244cd98abdeb19efb54f5f00f26f25ee89c2376b048c05e711b855eadf30a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-20T19:46:26.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:25 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-20T19:46:26.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:25 smithi118 systemd[1]: Stopped Ceph osd.7 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:46:26.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:25 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.7.service: Consumed 6min 27.618s CPU time. 2024-09-20T19:46:26.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 systemd[1]: Starting Ceph osd.7 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:46:26.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 247303 9.03 MB/sec execute 556 sec latency 67.060 ms 2024-09-20T19:46:26.500 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 podman[58951]: 2024-09-20 19:46:26.133413617 +0000 UTC m=+0.016145935 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:46:26.500 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 podman[58951]: 2024-09-20 19:46:26.262565026 +0000 UTC m=+0.145297337 container create 55539207e8d35594b32d3986fab6ef1d2fa742d18d09527c14df027334bc5e74 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:46:26.914 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 podman[58951]: 2024-09-20 19:46:26.498912825 +0000 UTC m=+0.381645152 container init 55539207e8d35594b32d3986fab6ef1d2fa742d18d09527c14df027334bc5e74 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-20T19:46:26.914 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 podman[58951]: 2024-09-20 19:46:26.502782405 +0000 UTC m=+0.385514716 container start 55539207e8d35594b32d3986fab6ef1d2fa742d18d09527c14df027334bc5e74 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:46:26.914 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 podman[58951]: 2024-09-20 19:46:26.590811901 +0000 UTC m=+0.473544222 container attach 55539207e8d35594b32d3986fab6ef1d2fa742d18d09527c14df027334bc5e74 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True) 2024-09-20T19:46:26.914 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:46:26.914 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 bash[58951]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:46:26.914 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:46:26.914 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:26 smithi118 bash[58951]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T19:46:26.968 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:26 smithi116 ceph-mon[63578]: pgmap v289: 97 pgs: 2 active+undersized, 3 stale+active+clean, 14 peering, 7 active+undersized+degraded, 71 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s wr, 330 op/s; 41/1695 objects degraded (2.419%) 2024-09-20T19:46:27.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:26 smithi118 ceph-mon[49197]: pgmap v289: 97 pgs: 2 active+undersized, 3 stale+active+clean, 14 peering, 7 active+undersized+degraded, 71 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s wr, 330 op/s; 41/1695 objects degraded (2.419%) 2024-09-20T19:46:27.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T19:46:27.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 bash[58951]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T19:46:27.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T19:46:27.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 bash[58951]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-20T19:46:27.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 247613 9.02 MB/sec execute 557 sec latency 53.309 ms 2024-09-20T19:46:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T19:46:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 bash[58951]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-20T19:46:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T19:46:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 bash[58951]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T19:46:27.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-20T19:46:27.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 bash[58951]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-20T19:46:27.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T19:46:27.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 bash[58951]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T19:46:27.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate[58966]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-20T19:46:27.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 bash[58951]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-20T19:46:27.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:27 smithi118 podman[58951]: 2024-09-20 19:46:27.421846212 +0000 UTC m=+1.304578531 container died 55539207e8d35594b32d3986fab6ef1d2fa742d18d09527c14df027334bc5e74 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, ceph=True) 2024-09-20T19:46:28.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 247829 9.01 MB/sec execute 558 sec latency 52.587 ms 2024-09-20T19:46:28.448 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 podman[59096]: 2024-09-20 19:46:28.095039659 +0000 UTC m=+0.667438004 container remove 55539207e8d35594b32d3986fab6ef1d2fa742d18d09527c14df027334bc5e74 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:46:28.448 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 podman[59154]: 2024-09-20 19:46:28.203338565 +0000 UTC m=+0.015739328 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:46:28.448 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 podman[59154]: 2024-09-20 19:46:28.329470324 +0000 UTC m=+0.141871117 container create d60081632e22113859dd3e0a3ecdec336dc904036c51171580053c7e52264e54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-20T19:46:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:28 smithi118 ceph-mon[49197]: pgmap v290: 97 pgs: 4 active+undersized, 14 peering, 19 active+undersized+degraded, 60 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 487 op/s; 117/1662 objects degraded (7.040%) 2024-09-20T19:46:28.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 podman[59154]: 2024-09-20 19:46:28.556774214 +0000 UTC m=+0.369174967 container init d60081632e22113859dd3e0a3ecdec336dc904036c51171580053c7e52264e54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:46:28.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 podman[59154]: 2024-09-20 19:46:28.560648853 +0000 UTC m=+0.373049601 container start d60081632e22113859dd3e0a3ecdec336dc904036c51171580053c7e52264e54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-20T19:46:28.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 ceph-osd[59173]: -- 172.21.15.118:0/3315515251 <== mon.1 v2:172.21.15.118:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e92adc61e0 con 0x55e92adc2000 2024-09-20T19:46:28.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 bash[59154]: d60081632e22113859dd3e0a3ecdec336dc904036c51171580053c7e52264e54 2024-09-20T19:46:28.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:28 smithi118 systemd[1]: Started Ceph osd.7 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:46:28.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:28.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:28.775 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:28 smithi116 ceph-mon[63578]: pgmap v290: 97 pgs: 4 active+undersized, 14 peering, 19 active+undersized+degraded, 60 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 487 op/s; 117/1662 objects degraded (7.040%) 2024-09-20T19:46:29.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 248051 9.00 MB/sec execute 559 sec latency 77.319 ms 2024-09-20T19:46:29.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:29 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:29.843 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:29 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:29.844 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:29 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:29.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:29 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:29.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:29 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:29.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:29 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:30.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 248325 9.01 MB/sec execute 560 sec latency 68.475 ms 2024-09-20T19:46:30.704 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:46:30.431+0000 7f99b2cbc640 -1 Falling back to public interface 2024-09-20T19:46:30.902 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:30 smithi116 ceph-mon[63578]: pgmap v291: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s wr, 382 op/s; 223/1656 objects degraded (13.466%) 2024-09-20T19:46:30.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:30 smithi118 ceph-mon[49197]: pgmap v291: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 345 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s wr, 382 op/s; 223/1656 objects degraded (13.466%) 2024-09-20T19:46:31.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 248853 9.02 MB/sec execute 561 sec latency 88.288 ms 2024-09-20T19:46:31.912 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:31 smithi116 ceph-mon[63578]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T19:46:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:31 smithi118 ceph-mon[49197]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T19:46:32.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 249438 9.04 MB/sec execute 562 sec latency 27.617 ms 2024-09-20T19:46:32.934 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:32 smithi116 ceph-mon[63578]: pgmap v292: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.4 MiB/s wr, 331 op/s; 223/1629 objects degraded (13.689%) 2024-09-20T19:46:32.934 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:32 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 223/1656 objects degraded (13.466%), 30 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:32.942 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:32 smithi118 ceph-mon[49197]: pgmap v292: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.4 MiB/s wr, 331 op/s; 223/1629 objects degraded (13.689%) 2024-09-20T19:46:32.943 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:32 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 223/1656 objects degraded (13.466%), 30 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:33.323 INFO:teuthology.orchestra.run.smithi116.stdout: 5 249695 9.03 MB/sec execute 563 sec latency 119.891 ms 2024-09-20T19:46:34.227 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:34.228 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:34.228 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:34.228 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:34 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:34.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 249901 9.02 MB/sec execute 564 sec latency 59.908 ms 2024-09-20T19:46:34.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:34.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:34.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:34.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:34 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:35.244 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:35 smithi116 ceph-mon[63578]: pgmap v293: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 350 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 417 op/s; 223/1725 objects degraded (12.928%) 2024-09-20T19:46:35.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 250178 9.02 MB/sec execute 565 sec latency 73.753 ms 2024-09-20T19:46:35.454 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:35 smithi118 ceph-mon[49197]: pgmap v293: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 350 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 417 op/s; 223/1725 objects degraded (12.928%) 2024-09-20T19:46:36.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 250438 9.02 MB/sec execute 566 sec latency 44.116 ms 2024-09-20T19:46:37.293 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:37 smithi116 ceph-mon[63578]: pgmap v294: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 411 op/s; 222/1731 objects degraded (12.825%) 2024-09-20T19:46:37.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 250751 9.02 MB/sec execute 567 sec latency 62.588 ms 2024-09-20T19:46:37.345 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:37 smithi118 ceph-mon[49197]: pgmap v294: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 411 op/s; 222/1731 objects degraded (12.825%) 2024-09-20T19:46:37.964 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:37 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:46:37.602+0000 7f99b2cbc640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-20T19:46:38.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 251060 9.01 MB/sec execute 568 sec latency 48.778 ms 2024-09-20T19:46:38.567 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.567 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.567 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 222/1731 objects degraded (12.825%), 30 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.568 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-20T19:46:38.569 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-20T19:46:38.570 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-20T19:46:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:38.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 222/1731 objects degraded (12.825%), 30 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-20T19:46:38.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-20T19:46:38.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-20T19:46:38.845 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:46:39.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:38 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:46:38.824+0000 7f99b2cbc640 -1 osd.7 153 log_to_monitors true 2024-09-20T19:46:39.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:46:39.019+0000 7f99aaa66640 -1 osd.7 153 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T19:46:39.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 251294 9.00 MB/sec execute 569 sec latency 60.775 ms 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: pgmap v295: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.8 MiB/s wr, 488 op/s; 223/1713 objects degraded (13.018%) 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all osd 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: Upgrade: Setting require_osd_release to 19 squid 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: from='osd.7 [v2:172.21.15.118:6824/1312213281,v1:172.21.15.118:6825/1312213281]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: from='osd.7 [v2:172.21.15.118:6824/1312213281,v1:172.21.15.118:6825/1312213281]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: osdmap e156: 8 total, 7 up, 8 in 2024-09-20T19:46:39.590 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:39.591 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:39 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi116.xvfgaf"]} : dispatch 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: pgmap v295: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.8 MiB/s wr, 488 op/s; 223/1713 objects degraded (13.018%) 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all osd 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: Upgrade: Setting require_osd_release to 19 squid 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: from='osd.7 [v2:172.21.15.118:6824/1312213281,v1:172.21.15.118:6825/1312213281]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: from='osd.7 [v2:172.21.15.118:6824/1312213281,v1:172.21.15.118:6825/1312213281]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:39.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: osdmap e156: 8 total, 7 up, 8 in 2024-09-20T19:46:39.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi118", "root=default"]} : dispatch 2024-09-20T19:46:39.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:39 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi116.xvfgaf"]} : dispatch 2024-09-20T19:46:39.831 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:46:39.831 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (9m) 2m ago 17m 14.7M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:46:39.831 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (5m) 2m ago 17m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:46:39.831 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (5m) 6s ago 17m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:46:39.831 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (7m) 2m ago 17m 47.1M - dad864ee21e9 8a163a77164a 2024-09-20T19:46:39.831 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (12m) 2m ago 11m 37.0M - 17.2.0 e1d6a67b021e c8aa09f7d92b 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (11m) 6s ago 11m 15.3M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (8m) 2m ago 19m 497M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (7m) 6s ago 17m 450M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (7m) 2m ago 19m 51.2M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (6m) 6s ago 17m 46.6M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (11m) 2m ago 11m 290M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (11m) 6s ago 11m 86.2M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (8m) 2m ago 17m 10.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (8m) 6s ago 17m 11.0M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (5m) 2m ago 16m 343M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (4m) 2m ago 15m 255M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (3m) 2m ago 15m 214M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (2m) 2m ago 15m 14.3M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (2m) 6s ago 14m 313M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (88s) 6s ago 14m 281M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:46:39.832 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (48s) 6s ago 13m 242M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 14b930edda7f 2024-09-20T19:46:39.833 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (11s) 6s ago 13m 14.3M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 d60081632e22 2024-09-20T19:46:39.833 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (7m) 2m ago 16m 55.4M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:46:40.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 251471 8.99 MB/sec execute 570 sec latency 298.186 ms 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:46:40.442 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 8 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 12 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:46:40.443 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:46:40.618 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:40 smithi116 ceph-mon[63578]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:40.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:40 smithi116 ceph-mon[63578]: Upgrade: It appears safe to stop mds.foofs.smithi116.xvfgaf 2024-09-20T19:46:40.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:40 smithi116 ceph-mon[63578]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:40.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:40 smithi116 ceph-mon[63578]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:46:40.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:40 smithi116 ceph-mon[63578]: osd.7 [v2:172.21.15.118:6824/1312213281,v1:172.21.15.118:6825/1312213281] boot 2024-09-20T19:46:40.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:40 smithi116 ceph-mon[63578]: osdmap e157: 8 total, 8 up, 8 in 2024-09-20T19:46:40.619 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:40 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T19:46:40.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:40 smithi118 ceph-mon[49197]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:40.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:40 smithi118 ceph-mon[49197]: Upgrade: It appears safe to stop mds.foofs.smithi116.xvfgaf 2024-09-20T19:46:40.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:40 smithi118 ceph-mon[49197]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:40.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:40 smithi118 ceph-mon[49197]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T19:46:40.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:40 smithi118 ceph-mon[49197]: osd.7 [v2:172.21.15.118:6824/1312213281,v1:172.21.15.118:6825/1312213281] boot 2024-09-20T19:46:40.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:40 smithi118 ceph-mon[49197]: osdmap e157: 8 total, 8 up, 8 in 2024-09-20T19:46:40.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:40 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T19:46:40.833 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "crash", 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "osd" 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "14/23 daemons upgraded", 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading mds daemons", 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:46:40.834 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:46:41.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 251631 8.98 MB/sec execute 571 sec latency 1089.841 ms 2024-09-20T19:46:41.395 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); Degraded data redundancy: 211/1647 objects degraded (12.811%), 30 pgs degraded; 5 mgr modules have recently crashed 2024-09-20T19:46:41.395 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:46:41.395 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:46:41.395 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 211/1647 objects degraded (12.811%), 30 pgs degraded 2024-09-20T19:46:41.395 INFO:teuthology.orchestra.run.smithi116.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-09-20T19:46:41.396 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.f is active+undersized+degraded, acting [2,0] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-09-20T19:46:41.397 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:46:41.398 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:46:41.398 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:46:41.398 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:46:41.398 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:46:41.398 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:46:41.656 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: pgmap v297: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 419 op/s; 211/1647 objects degraded (12.811%) 2024-09-20T19:46:41.656 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:41.656 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/270986356' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:41.657 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:41.657 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:46:41.657 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:41.657 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: osdmap e158: 8 total, 8 up, 8 in 2024-09-20T19:46:41.657 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:41 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/4061597023' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:46:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: pgmap v297: 97 pgs: 7 active+undersized, 30 active+undersized+degraded, 60 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 419 op/s; 211/1647 objects degraded (12.811%) 2024-09-20T19:46:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/270986356' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:46:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi116.xvfgaf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:46:41.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:46:41.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: osdmap e158: 8 total, 8 up, 8 in 2024-09-20T19:46:41.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:41 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/4061597023' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:46:42.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 251767 8.98 MB/sec execute 572 sec latency 887.734 ms 2024-09-20T19:46:42.561 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:42 smithi116 ceph-mon[63578]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:42.561 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:42 smithi116 ceph-mon[63578]: Upgrade: Updating mds.foofs.smithi116.xvfgaf 2024-09-20T19:46:42.561 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:42 smithi116 ceph-mon[63578]: Deploying daemon mds.foofs.smithi116.xvfgaf on smithi116 2024-09-20T19:46:42.561 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:42 smithi116 ceph-mon[63578]: osdmap e159: 8 total, 8 up, 8 in 2024-09-20T19:46:42.964 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:42 smithi118 ceph-mon[49197]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:46:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:42 smithi118 ceph-mon[49197]: Upgrade: Updating mds.foofs.smithi116.xvfgaf 2024-09-20T19:46:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:42 smithi118 ceph-mon[49197]: Deploying daemon mds.foofs.smithi116.xvfgaf on smithi116 2024-09-20T19:46:42.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:42 smithi118 ceph-mon[49197]: osdmap e159: 8 total, 8 up, 8 in 2024-09-20T19:46:43.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252051 8.97 MB/sec execute 573 sec latency 63.998 ms 2024-09-20T19:46:43.767 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:43 smithi116 ceph-mon[63578]: pgmap v300: 97 pgs: 7 peering, 5 active+undersized, 21 active+undersized+degraded, 64 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.8 MiB/s wr, 343 op/s; 173/1629 objects degraded (10.620%); 0 B/s, 0 objects/s recovering 2024-09-20T19:46:43.767 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:43 smithi116 ceph-mon[63578]: Health check update: Degraded data redundancy: 173/1629 objects degraded (10.620%), 21 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:43.768 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:43 smithi116 ceph-mon[63578]: osdmap e160: 8 total, 8 up, 8 in 2024-09-20T19:46:43.768 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:43.768 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:43.768 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:43 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[63574]: 2024-09-20T19:46:43.691+0000 7fdd9ed3c640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:46:43.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:43 smithi118 ceph-mon[49197]: pgmap v300: 97 pgs: 7 peering, 5 active+undersized, 21 active+undersized+degraded, 64 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.8 MiB/s wr, 343 op/s; 173/1629 objects degraded (10.620%); 0 B/s, 0 objects/s recovering 2024-09-20T19:46:43.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:43 smithi118 ceph-mon[49197]: Health check update: Degraded data redundancy: 173/1629 objects degraded (10.620%), 21 pgs degraded (PG_DEGRADED) 2024-09-20T19:46:43.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:43 smithi118 ceph-mon[49197]: osdmap e160: 8 total, 8 up, 8 in 2024-09-20T19:46:43.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:43.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:44.123 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:46:43 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:46:43.765+0000 7effe56e4640 -1 bdev(0x561a9c19ac00 /var/lib/ceph/osd/ceph-2/block) aio_submit retries 3 2024-09-20T19:46:44.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252131 8.96 MB/sec execute 574 sec latency 655.805 ms 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: osdmap e161: 8 total, 8 up, 8 in 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: osdmap e162: 8 total, 8 up, 8 in 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: Standby daemon mds.foofs.smithi118.zumcyi assigned to filesystem foofs as rank 0 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T19:46:44.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:44 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:replay} 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: osdmap e161: 8 total, 8 up, 8 in 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: osdmap e162: 8 total, 8 up, 8 in 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: Standby daemon mds.foofs.smithi118.zumcyi assigned to filesystem foofs as rank 0 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T19:46:44.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:44 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:replay} 2024-09-20T19:46:45.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252131 8.95 MB/sec execute 575 sec latency 1655.833 ms 2024-09-20T19:46:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:45 smithi116 ceph-mon[63578]: pgmap v305: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 9 active+undersized+degraded, 80 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.8 MiB/s wr, 483 op/s; 87/1560 objects degraded (5.577%); 557 KiB/s, 42 objects/s recovering 2024-09-20T19:46:45.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:45 smithi118 ceph-mon[49197]: pgmap v305: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 7 peering, 9 active+undersized+degraded, 80 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.8 MiB/s wr, 483 op/s; 87/1560 objects degraded (5.577%); 557 KiB/s, 42 objects/s recovering 2024-09-20T19:46:46.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252131 8.93 MB/sec execute 576 sec latency 2655.858 ms 2024-09-20T19:46:46.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:46 smithi116 ceph-mon[63578]: reconnect by client.24309 172.21.15.118:0/1616532918 after 0 2024-09-20T19:46:46.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:46 smithi116 ceph-mon[63578]: reconnect by client.14552 172.21.15.116:0/1227733714 after 0 2024-09-20T19:46:46.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:46 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:reconnect 2024-09-20T19:46:46.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:46 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:reconnect} 2024-09-20T19:46:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:46 smithi118 ceph-mon[49197]: reconnect by client.24309 172.21.15.118:0/1616532918 after 0 2024-09-20T19:46:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:46 smithi118 ceph-mon[49197]: reconnect by client.14552 172.21.15.116:0/1227733714 after 0 2024-09-20T19:46:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:46 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:reconnect 2024-09-20T19:46:46.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:46 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:reconnect} 2024-09-20T19:46:47.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252131 8.91 MB/sec execute 577 sec latency 3655.916 ms 2024-09-20T19:46:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:47 smithi116 ceph-mon[63578]: pgmap v306: 97 pgs: 97 active+clean; 343 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 4.0 MiB/s wr, 393 op/s; 887 KiB/s, 61 objects/s recovering 2024-09-20T19:46:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:47 smithi116 ceph-mon[63578]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 87/1560 objects degraded (5.577%), 10 pgs degraded) 2024-09-20T19:46:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:47 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:rejoin 2024-09-20T19:46:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:47 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:rejoin} 2024-09-20T19:46:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:47 smithi118 ceph-mon[49197]: pgmap v306: 97 pgs: 97 active+clean; 343 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 4.0 MiB/s wr, 393 op/s; 887 KiB/s, 61 objects/s recovering 2024-09-20T19:46:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:47 smithi118 ceph-mon[49197]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 87/1560 objects degraded (5.577%), 10 pgs degraded) 2024-09-20T19:46:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:47 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:rejoin 2024-09-20T19:46:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:47 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:rejoin} 2024-09-20T19:46:48.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252131 8.90 MB/sec execute 578 sec latency 4655.974 ms 2024-09-20T19:46:48.749 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:48 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:clientreplay 2024-09-20T19:46:48.750 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:48 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:clientreplay} 2024-09-20T19:46:48.750 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:48 smithi116 ceph-mon[63578]: daemon mds.foofs.smithi118.zumcyi is now active in filesystem foofs as rank 0 2024-09-20T19:46:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:48 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:clientreplay 2024-09-20T19:46:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:48 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi118.zumcyi=up:clientreplay} 2024-09-20T19:46:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:48 smithi118 ceph-mon[49197]: daemon mds.foofs.smithi118.zumcyi is now active in filesystem foofs as rank 0 2024-09-20T19:46:49.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252503 8.91 MB/sec execute 579 sec latency 4829.805 ms 2024-09-20T19:46:49.519 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:49 smithi116 ceph-mon[63578]: pgmap v307: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 3.3 MiB/s wr, 361 op/s; 613 KiB/s, 42 objects/s recovering 2024-09-20T19:46:49.519 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:49 smithi116 ceph-mon[63578]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T19:46:49.519 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:49 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:active 2024-09-20T19:46:49.519 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:49 smithi116 ceph-mon[63578]: fsmap foofs:1 {0=foofs.smithi118.zumcyi=up:active} 2024-09-20T19:46:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:49 smithi118 ceph-mon[49197]: pgmap v307: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 3.3 MiB/s wr, 361 op/s; 613 KiB/s, 42 objects/s recovering 2024-09-20T19:46:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:49 smithi118 ceph-mon[49197]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T19:46:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:49 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.118:6832/844431701,v1:172.21.15.118:6833/844431701] up:active 2024-09-20T19:46:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:49 smithi118 ceph-mon[49197]: fsmap foofs:1 {0=foofs.smithi118.zumcyi=up:active} 2024-09-20T19:46:50.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 252863 8.91 MB/sec execute 580 sec latency 75.083 ms 2024-09-20T19:46:51.324 INFO:teuthology.orchestra.run.smithi116.stdout: 5 253032 8.91 MB/sec execute 581 sec latency 120.094 ms 2024-09-20T19:46:51.578 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:51 smithi116 ceph-mon[63578]: pgmap v308: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 1.5 MiB/s wr, 129 op/s; 392 KiB/s, 27 objects/s recovering 2024-09-20T19:46:51.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:51 smithi118 ceph-mon[49197]: pgmap v308: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 1.5 MiB/s wr, 129 op/s; 392 KiB/s, 27 objects/s recovering 2024-09-20T19:46:52.329 INFO:teuthology.orchestra.run.smithi116.stdout: 5 253300 8.90 MB/sec execute 582 sec latency 157.050 ms 2024-09-20T19:46:53.329 INFO:teuthology.orchestra.run.smithi116.stdout: 5 253415 8.89 MB/sec execute 583 sec latency 99.799 ms 2024-09-20T19:46:53.678 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:53 smithi116 ceph-mon[63578]: pgmap v309: 97 pgs: 97 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 1.9 MiB/s wr, 222 op/s; 345 KiB/s, 24 objects/s recovering 2024-09-20T19:46:53.678 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:53 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:53.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:53 smithi118 ceph-mon[49197]: pgmap v309: 97 pgs: 97 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 1.9 MiB/s wr, 222 op/s; 345 KiB/s, 24 objects/s recovering 2024-09-20T19:46:53.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:53 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:54.329 INFO:teuthology.orchestra.run.smithi116.stdout: 5 253590 8.89 MB/sec execute 584 sec latency 149.513 ms 2024-09-20T19:46:55.329 INFO:teuthology.orchestra.run.smithi116.stdout: 5 253906 8.88 MB/sec execute 585 sec latency 50.022 ms 2024-09-20T19:46:55.724 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:55 smithi116 ceph-mon[63578]: pgmap v310: 97 pgs: 97 active+clean; 353 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.4 MiB/s wr, 255 op/s; 197 KiB/s, 12 objects/s recovering 2024-09-20T19:46:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:55 smithi118 ceph-mon[49197]: pgmap v310: 97 pgs: 97 active+clean; 353 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 2.4 MiB/s wr, 255 op/s; 197 KiB/s, 12 objects/s recovering 2024-09-20T19:46:56.329 INFO:teuthology.orchestra.run.smithi116.stdout: 5 254165 8.88 MB/sec execute 586 sec latency 127.501 ms 2024-09-20T19:46:57.329 INFO:teuthology.orchestra.run.smithi116.stdout: 5 254741 8.89 MB/sec execute 587 sec latency 37.921 ms 2024-09-20T19:46:57.526 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:57 smithi116 ceph-mon[63578]: pgmap v311: 97 pgs: 97 active+clean; 353 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.5 MiB/s wr, 238 op/s; 166 KiB/s, 10 objects/s recovering 2024-09-20T19:46:57.526 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:57.526 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:57.526 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:57 smithi118 ceph-mon[49197]: pgmap v311: 97 pgs: 97 active+clean; 353 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.5 MiB/s wr, 238 op/s; 166 KiB/s, 10 objects/s recovering 2024-09-20T19:46:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:46:58.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 255290 8.89 MB/sec execute 588 sec latency 40.061 ms 2024-09-20T19:46:58.543 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:58 smithi116 ceph-mon[63578]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T19:46:58.544 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:58 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:boot 2024-09-20T19:46:58.544 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:58 smithi116 ceph-mon[63578]: fsmap foofs:1 {0=foofs.smithi118.zumcyi=up:active} 1 up:standby 2024-09-20T19:46:58.544 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"} : dispatch 2024-09-20T19:46:58.544 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:58.544 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:58 smithi118 ceph-mon[49197]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T19:46:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:58 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:boot 2024-09-20T19:46:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:58 smithi118 ceph-mon[49197]: fsmap foofs:1 {0=foofs.smithi118.zumcyi=up:active} 1 up:standby 2024-09-20T19:46:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi116.xvfgaf"} : dispatch 2024-09-20T19:46:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:46:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:46:59.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 255488 8.88 MB/sec execute 589 sec latency 104.209 ms 2024-09-20T19:46:59.595 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:46:59 smithi116 ceph-mon[63578]: pgmap v312: 97 pgs: 97 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.4 MiB/s wr, 239 op/s 2024-09-20T19:46:59.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:46:59 smithi118 ceph-mon[49197]: pgmap v312: 97 pgs: 97 active+clean; 351 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.4 MiB/s wr, 239 op/s 2024-09-20T19:47:00.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 255726 8.88 MB/sec execute 590 sec latency 80.190 ms 2024-09-20T19:47:01.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 256020 8.89 MB/sec execute 591 sec latency 84.723 ms 2024-09-20T19:47:01.700 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:01 smithi116 ceph-mon[63578]: pgmap v313: 97 pgs: 97 active+clean; 359 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 717 KiB/s rd, 3.5 MiB/s wr, 369 op/s 2024-09-20T19:47:01.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:01 smithi118 ceph-mon[49197]: pgmap v313: 97 pgs: 97 active+clean; 359 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 717 KiB/s rd, 3.5 MiB/s wr, 369 op/s 2024-09-20T19:47:02.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 256192 8.88 MB/sec execute 592 sec latency 87.952 ms 2024-09-20T19:47:03.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 256434 8.87 MB/sec execute 593 sec latency 102.651 ms 2024-09-20T19:47:03.784 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:03 smithi116 ceph-mon[63578]: pgmap v314: 97 pgs: 97 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 3.9 MiB/s wr, 443 op/s 2024-09-20T19:47:03.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:03 smithi118 ceph-mon[49197]: pgmap v314: 97 pgs: 97 active+clean; 355 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 3.9 MiB/s wr, 443 op/s 2024-09-20T19:47:04.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 256620 8.88 MB/sec execute 594 sec latency 105.735 ms 2024-09-20T19:47:05.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 256763 8.87 MB/sec execute 595 sec latency 98.630 ms 2024-09-20T19:47:05.595 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:05 smithi116 ceph-mon[63578]: pgmap v315: 97 pgs: 97 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.3 MiB/s wr, 402 op/s 2024-09-20T19:47:05.596 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:05 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:05.596 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:05 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:05.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:05 smithi118 ceph-mon[49197]: pgmap v315: 97 pgs: 97 active+clean; 350 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.3 MiB/s wr, 402 op/s 2024-09-20T19:47:05.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:05 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:05.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:05 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:06.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 256904 8.86 MB/sec execute 596 sec latency 120.204 ms 2024-09-20T19:47:06.899 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:06 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:06.900 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:06 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:06.900 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:06 smithi116 ceph-mon[63578]: pgmap v316: 97 pgs: 97 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.2 MiB/s wr, 428 op/s 2024-09-20T19:47:07.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:06 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:07.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:06 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:07.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:06 smithi118 ceph-mon[49197]: pgmap v316: 97 pgs: 97 active+clean; 356 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.2 MiB/s wr, 428 op/s 2024-09-20T19:47:07.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 257062 8.85 MB/sec execute 597 sec latency 78.312 ms 2024-09-20T19:47:08.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 257323 8.85 MB/sec execute 598 sec latency 79.888 ms 2024-09-20T19:47:08.957 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:08 smithi116 ceph-mon[63578]: pgmap v317: 97 pgs: 97 active+clean; 354 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.1 MiB/s wr, 417 op/s 2024-09-20T19:47:09.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:08 smithi118 ceph-mon[49197]: pgmap v317: 97 pgs: 97 active+clean; 354 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.1 MiB/s wr, 417 op/s 2024-09-20T19:47:09.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 257691 8.85 MB/sec execute 599 sec latency 38.763 ms 2024-09-20T19:47:10.330 INFO:teuthology.orchestra.run.smithi116.stdout: 5 cleanup 600 sec 2024-09-20T19:47:10.574 INFO:teuthology.orchestra.run.smithi116.stdout: 0 cleanup 600 sec 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Operation Count AvgLat MaxLat 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: ---------------------------------------- 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: NTCreateX 170496 4.957 4829.802 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Close 125156 0.129 35.043 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Rename 7220 9.424 2247.822 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Unlink 34457 7.493 2447.860 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Qpathinfo 154552 1.296 2444.855 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Qfileinfo 26933 0.006 0.709 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Qfsinfo 28314 0.972 2121.513 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Sfileinfo 13905 14.273 4812.353 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: Find 59692 0.907 79.883 2024-09-20T19:47:10.575 INFO:teuthology.orchestra.run.smithi116.stdout: WriteX 84173 15.750 4823.470 2024-09-20T19:47:10.576 INFO:teuthology.orchestra.run.smithi116.stdout: ReadX 267131 0.010 6.686 2024-09-20T19:47:10.576 INFO:teuthology.orchestra.run.smithi116.stdout: LockX 554 0.521 20.449 2024-09-20T19:47:10.576 INFO:teuthology.orchestra.run.smithi116.stdout: UnlockX 554 0.439 21.608 2024-09-20T19:47:10.576 INFO:teuthology.orchestra.run.smithi116.stdout: Flush 11955 0.002 0.044 2024-09-20T19:47:10.576 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:47:10.576 INFO:teuthology.orchestra.run.smithi116.stdout:Throughput 8.84942 MB/sec 5 clients 5 procs max_latency=4829.805 ms 2024-09-20T19:47:10.577 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-20T19:47:10.612 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-20T19:47:10.730 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-20T19:47:10.777 INFO:teuthology.orchestra.run.smithi116.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-20T19:47:10.777 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:47:10.777 INFO:teuthology.orchestra.run.smithi116.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-20T19:47:10.777 INFO:teuthology.orchestra.run.smithi116.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-20T19:47:10.877 INFO:teuthology.orchestra.run.smithi116.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-20T19:47:10.877 INFO:teuthology.orchestra.run.smithi116.stdout:releasing clients 2024-09-20T19:47:10.990 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:10 smithi116 ceph-mon[63578]: pgmap v318: 97 pgs: 97 active+clean; 354 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 411 op/s 2024-09-20T19:47:11.214 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:10 smithi118 ceph-mon[49197]: pgmap v318: 97 pgs: 97 active+clean; 354 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 411 op/s 2024-09-20T19:47:11.914 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:47:12.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: Detected new or changed devices on smithi116 2024-09-20T19:47:12.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: pgmap v319: 97 pgs: 97 active+clean; 360 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 3.4 MiB/s wr, 298 op/s 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi118.zumcyi"]} : dispatch 2024-09-20T19:47:12.790 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: Upgrade: It appears safe to stop mds.foofs.smithi118.zumcyi 2024-09-20T19:47:12.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:12.791 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: Detected new or changed devices on smithi116 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: pgmap v319: 97 pgs: 97 active+clean; 360 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 3.4 MiB/s wr, 298 op/s 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:12.823 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi118.zumcyi"]} : dispatch 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: Upgrade: It appears safe to stop mds.foofs.smithi118.zumcyi 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:12.824 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:12.877 INFO:teuthology.orchestra.run.smithi116.stdout: 5 369 21.59 MB/sec execute 1 sec latency 67.866 ms 2024-09-20T19:47:12.920 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:47:12.920 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (9m) 7s ago 18m 20.0M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:47:12.920 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (6m) 7s ago 18m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:47:12.920 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (6m) 39s ago 17m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:47:12.920 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (8m) 7s ago 18m 46.4M - dad864ee21e9 8a163a77164a 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (16s) 7s ago 12m 10.8M - 19.3.0-5127-ge8baca1e 82f5f45939a4 587a46003f50 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (12m) 39s ago 12m 15.3M - 17.2.0 e1d6a67b021e 02398850bdd6 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (8m) 7s ago 19m 502M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (8m) 39s ago 17m 450M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (7m) 7s ago 19m 60.6M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (7m) 39s ago 17m 46.6M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (12m) 7s ago 12m 287M - 4.0 e1d6a67b021e 7288f3493d81 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (12m) 39s ago 12m 86.2M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (9m) 7s ago 18m 11.9M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (9m) 39s ago 17m 11.0M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (5m) 7s ago 17m 401M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (4m) 7s ago 16m 439M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (4m) 7s ago 16m 358M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (3m) 7s ago 15m 457M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:47:12.921 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (2m) 39s ago 15m 313M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:47:12.922 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (2m) 39s ago 14m 281M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:47:12.922 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (81s) 39s ago 14m 242M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 14b930edda7f 2024-09-20T19:47:12.922 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (44s) 39s ago 14m 14.3M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 d60081632e22 2024-09-20T19:47:12.922 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (8m) 7s ago 17m 55.6M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:13.614 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 8 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 1 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 13 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:47:13.615 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:47:13.860 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:13 smithi116 ceph-mon[63578]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:13.860 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:13.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 507 21.88 MB/sec execute 2 sec latency 57.773 ms 2024-09-20T19:47:13.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:13 smithi118 ceph-mon[49197]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:13.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "mon", 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "crash", 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: "osd" 2024-09-20T19:47:14.056 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:47:14.057 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "15/23 daemons upgraded", 2024-09-20T19:47:14.057 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading mds daemons", 2024-09-20T19:47:14.057 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:47:14.057 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:47:14.593 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:47:14.593 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:47:14.593 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:47:14.593 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:47:14.593 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:47:14.594 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:47:14.594 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:47:14.594 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:47:14.594 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:47:14.776 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: Upgrade: Updating mds.foofs.smithi118.zumcyi 2024-09-20T19:47:14.777 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2299553017' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:14.777 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:47:14.777 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:14.777 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: Deploying daemon mds.foofs.smithi118.zumcyi on smithi118 2024-09-20T19:47:14.777 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: pgmap v320: 97 pgs: 97 active+clean; 365 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.7 MiB/s wr, 335 op/s 2024-09-20T19:47:14.777 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:14.777 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:14 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2979061417' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:47:14.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 735 20.81 MB/sec execute 3 sec latency 90.905 ms 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: Upgrade: Updating mds.foofs.smithi118.zumcyi 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2299553017' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi118.zumcyi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: Deploying daemon mds.foofs.smithi118.zumcyi on smithi118 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: pgmap v320: 97 pgs: 97 active+clean; 365 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.7 MiB/s wr, 335 op/s 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:14.963 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:14 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2979061417' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:47:15.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 1175 17.74 MB/sec execute 4 sec latency 26.343 ms 2024-09-20T19:47:16.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[63574]: 2024-09-20T19:47:16.733+0000 7fdd9ed3c640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:47:16.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 cleanup 5 sec 2024-09-20T19:47:17.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:47:16.761+0000 7f9992a1c640 -1 bdev(0x55e92ae1cc00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 4 2024-09-20T19:47:17.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:47:16.763+0000 7f9996a24640 -1 bdev(0x55e92ae1cc00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 5 2024-09-20T19:47:17.216 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:47:16.755+0000 7fece2897640 -1 bdev(0x5559a4f88c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 1 2024-09-20T19:47:17.216 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:47:16.757+0000 7fece1895640 -1 bdev(0x5559a4f88c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-20T19:47:17.216 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:47:16.757+0000 7fece1094640 -1 bdev(0x5559a4f88c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: pgmap v321: 97 pgs: 97 active+clean; 359 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 9.7 MiB/s wr, 448 op/s 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: osdmap e163: 8 total, 8 up, 8 in 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: Standby daemon mds.foofs.smithi116.xvfgaf assigned to filesystem foofs as rank 0 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T19:47:17.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:16 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:replay} 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: pgmap v321: 97 pgs: 97 active+clean; 359 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 9.7 MiB/s wr, 448 op/s 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: osdmap e163: 8 total, 8 up, 8 in 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: Standby daemon mds.foofs.smithi116.xvfgaf assigned to filesystem foofs as rank 0 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T19:47:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:16 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:replay} 2024-09-20T19:47:17.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 cleanup 6 sec 2024-09-20T19:47:18.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 cleanup 7 sec 2024-09-20T19:47:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:18 smithi118 ceph-mon[49197]: pgmap v323: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 12 MiB/s wr, 524 op/s 2024-09-20T19:47:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:18 smithi118 ceph-mon[49197]: reconnect by client.24309 172.21.15.118:0/1616532918 after 0.000999984 2024-09-20T19:47:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:18 smithi118 ceph-mon[49197]: reconnect by client.14552 172.21.15.116:0/1227733714 after 0.000999984 2024-09-20T19:47:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:18 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:reconnect 2024-09-20T19:47:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:18 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:reconnect} 2024-09-20T19:47:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:18 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:47:19.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:18 smithi116 ceph-mon[63578]: pgmap v323: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 12 MiB/s wr, 524 op/s 2024-09-20T19:47:19.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:18 smithi116 ceph-mon[63578]: reconnect by client.24309 172.21.15.118:0/1616532918 after 0.000999984 2024-09-20T19:47:19.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:18 smithi116 ceph-mon[63578]: reconnect by client.14552 172.21.15.116:0/1227733714 after 0.000999984 2024-09-20T19:47:19.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:18 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:reconnect 2024-09-20T19:47:19.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:18 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:reconnect} 2024-09-20T19:47:19.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:18 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:47:19.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 cleanup 8 sec 2024-09-20T19:47:20.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:19 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:rejoin 2024-09-20T19:47:20.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:19 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:rejoin} 2024-09-20T19:47:20.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:19 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:rejoin 2024-09-20T19:47:20.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:19 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:rejoin} 2024-09-20T19:47:20.878 INFO:teuthology.orchestra.run.smithi116.stdout: 5 cleanup 9 sec 2024-09-20T19:47:21.141 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:20 smithi116 ceph-mon[63578]: pgmap v324: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 13 MiB/s wr, 528 op/s 2024-09-20T19:47:21.141 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:20 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:clientreplay 2024-09-20T19:47:21.142 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:20 smithi116 ceph-mon[63578]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:clientreplay} 2024-09-20T19:47:21.142 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:20 smithi116 ceph-mon[63578]: daemon mds.foofs.smithi116.xvfgaf is now active in filesystem foofs as rank 0 2024-09-20T19:47:21.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:20 smithi118 ceph-mon[49197]: pgmap v324: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 13 MiB/s wr, 528 op/s 2024-09-20T19:47:21.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:20 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:clientreplay 2024-09-20T19:47:21.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:20 smithi118 ceph-mon[49197]: fsmap foofs:1/1 {0=foofs.smithi116.xvfgaf=up:clientreplay} 2024-09-20T19:47:21.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:20 smithi118 ceph-mon[49197]: daemon mds.foofs.smithi116.xvfgaf is now active in filesystem foofs as rank 0 2024-09-20T19:47:21.221 INFO:teuthology.orchestra.run.smithi116.stdout: 0 cleanup 9 sec 2024-09-20T19:47:21.221 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:47:21.221 INFO:teuthology.orchestra.run.smithi116.stdout: Operation Count AvgLat MaxLat 2024-09-20T19:47:21.221 INFO:teuthology.orchestra.run.smithi116.stdout: ---------------------------------------- 2024-09-20T19:47:21.221 INFO:teuthology.orchestra.run.smithi116.stdout: NTCreateX 1119 9.786 4231.149 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Close 937 14.850 4173.918 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Rename 50 9.792 32.901 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Unlink 106 4.527 84.745 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Qpathinfo 730 1.334 19.371 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Qfileinfo 401 25.833 4166.604 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Qfsinfo 165 0.499 3.911 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Sfileinfo 170 11.273 82.520 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Find 301 0.836 3.137 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: WriteX 1721 0.024 0.106 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: ReadX 1271 0.015 0.496 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: Flush 132 43.304 4164.686 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:47:21.222 INFO:teuthology.orchestra.run.smithi116.stdout:Throughput 17.7388 MB/sec 5 clients 5 procs max_latency=90.905 ms 2024-09-20T19:47:21.223 DEBUG:teuthology.parallel:result is None 2024-09-20T19:47:22.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:21 smithi116 ceph-mon[63578]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T19:47:22.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:21 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:active 2024-09-20T19:47:22.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:21 smithi116 ceph-mon[63578]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 2024-09-20T19:47:22.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:21 smithi118 ceph-mon[49197]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T19:47:22.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:21 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497] up:active 2024-09-20T19:47:22.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:21 smithi118 ceph-mon[49197]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 2024-09-20T19:47:23.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:22 smithi118 ceph-mon[49197]: pgmap v325: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 12 MiB/s wr, 472 op/s 2024-09-20T19:47:23.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:22 smithi116 ceph-mon[63578]: pgmap v325: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 12 MiB/s wr, 472 op/s 2024-09-20T19:47:25.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:24 smithi118 ceph-mon[49197]: pgmap v326: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.2 MiB/s wr, 358 op/s 2024-09-20T19:47:25.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:24 smithi116 ceph-mon[63578]: pgmap v326: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.2 MiB/s wr, 358 op/s 2024-09-20T19:47:27.162 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:26 smithi118 ceph-mon[49197]: pgmap v327: 97 pgs: 97 active+clean; 295 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.7 MiB/s wr, 207 op/s 2024-09-20T19:47:27.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:26 smithi116 ceph-mon[63578]: pgmap v327: 97 pgs: 97 active+clean; 295 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.7 MiB/s wr, 207 op/s 2024-09-20T19:47:28.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:28.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:28.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:28.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:28.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:29.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:29 smithi116 ceph-mon[63578]: pgmap v328: 97 pgs: 97 active+clean; 267 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.7 MiB/s wr, 187 op/s 2024-09-20T19:47:29.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:29 smithi116 ceph-mon[63578]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T19:47:29.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:29 smithi116 ceph-mon[63578]: mds.? [v2:172.21.15.118:6832/1815698944,v1:172.21.15.118:6833/1815698944] up:boot 2024-09-20T19:47:29.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:29 smithi116 ceph-mon[63578]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:47:29.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:29 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:47:29.912 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:29 smithi118 ceph-mon[49197]: pgmap v328: 97 pgs: 97 active+clean; 267 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.7 MiB/s wr, 187 op/s 2024-09-20T19:47:29.912 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:29 smithi118 ceph-mon[49197]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T19:47:29.912 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:29 smithi118 ceph-mon[49197]: mds.? [v2:172.21.15.118:6832/1815698944,v1:172.21.15.118:6833/1815698944] up:boot 2024-09-20T19:47:29.912 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:29 smithi118 ceph-mon[49197]: fsmap foofs:1 {0=foofs.smithi116.xvfgaf=up:active} 1 up:standby 2024-09-20T19:47:29.912 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:29 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "mds metadata", "who": "foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:47:30.847 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:30 smithi118 ceph-mon[49197]: pgmap v329: 97 pgs: 97 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.3 MiB/s wr, 92 op/s 2024-09-20T19:47:30.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:30 smithi116 ceph-mon[63578]: pgmap v329: 97 pgs: 97 active+clean; 233 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.3 MiB/s wr, 92 op/s 2024-09-20T19:47:32.400 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:32 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:32.400 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:32 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:32.400 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:32 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:32.400 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:32 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:32.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:32 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:32.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:32 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:32.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:32 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:32.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:32 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:33.271 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:33 smithi118 ceph-mon[49197]: pgmap v330: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 382 KiB/s wr, 75 op/s 2024-09-20T19:47:33.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:33 smithi116 ceph-mon[63578]: pgmap v330: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 382 KiB/s wr, 75 op/s 2024-09-20T19:47:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:35 smithi118 ceph-mon[49197]: pgmap v331: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 232 KiB/s wr, 63 op/s 2024-09-20T19:47:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:47:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:35.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:35 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:35.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:35 smithi116 ceph-mon[63578]: pgmap v331: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 232 KiB/s wr, 63 op/s 2024-09-20T19:47:35.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:35.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:35.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:35.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:47:35.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:35.599 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:35 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: Detected new or changed devices on smithi118 2024-09-20T19:47:36.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:36.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all mds 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi116.xvfgaf"} : dispatch 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi116.xvfgaf"}]': finished 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.zumcyi"}]': finished 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all rgw 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.466 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all rbd-mirror 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all ceph-exporter 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all iscsi 2024-09-20T19:47:36.467 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:36 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: Detected new or changed devices on smithi118 2024-09-20T19:47:36.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all mds 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi116.xvfgaf"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi116.xvfgaf"}]': finished 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.zumcyi"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi118.zumcyi"}]': finished 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all rgw 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all rbd-mirror 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all ceph-exporter 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all iscsi 2024-09-20T19:47:36.625 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:36 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:37.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:37 smithi116 ceph-mon[63578]: pgmap v332: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 69 KiB/s wr, 42 op/s 2024-09-20T19:47:37.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:37 smithi118 ceph-mon[49197]: pgmap v332: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 69 KiB/s wr, 42 op/s 2024-09-20T19:47:38.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: Upgrade: Updating nfs.foo.0.0.smithi116.bfcxla (1/2) 2024-09-20T19:47:38.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:38.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla 2024-09-20T19:47:38.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T19:47:38.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T19:47:38.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T19:47:38.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:47:38.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:38.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T19:47:38.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:47:38.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T19:47:38.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:38 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: Upgrade: Updating nfs.foo.0.0.smithi116.bfcxla (1/2) 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi116.bfcxla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T19:47:38.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:38 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:39.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:39 smithi116 ceph-mon[63578]: Rados config object exists: conf-nfs.foo 2024-09-20T19:47:39.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:39 smithi116 ceph-mon[63578]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla-rgw 2024-09-20T19:47:39.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:39 smithi116 ceph-mon[63578]: Bind address in nfs.foo.0.0.smithi116.bfcxla's ganesha conf is defaulting to empty 2024-09-20T19:47:39.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:39 smithi116 ceph-mon[63578]: Deploying daemon nfs.foo.0.0.smithi116.bfcxla on smithi116 2024-09-20T19:47:39.624 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:39 smithi116 ceph-mon[63578]: pgmap v333: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 28 KiB/s wr, 20 op/s 2024-09-20T19:47:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:39 smithi118 ceph-mon[49197]: Rados config object exists: conf-nfs.foo 2024-09-20T19:47:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:39 smithi118 ceph-mon[49197]: Creating key for client.nfs.foo.0.0.smithi116.bfcxla-rgw 2024-09-20T19:47:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:39 smithi118 ceph-mon[49197]: Bind address in nfs.foo.0.0.smithi116.bfcxla's ganesha conf is defaulting to empty 2024-09-20T19:47:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:39 smithi118 ceph-mon[49197]: Deploying daemon nfs.foo.0.0.smithi116.bfcxla on smithi116 2024-09-20T19:47:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:39 smithi118 ceph-mon[49197]: pgmap v333: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 28 KiB/s wr, 20 op/s 2024-09-20T19:47:41.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:41 smithi116 ceph-mon[63578]: pgmap v334: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 25 KiB/s wr, 11 op/s 2024-09-20T19:47:41.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:41 smithi116 ceph-mon[63578]: from='mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497]' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.116:0/1227733714"} : dispatch 2024-09-20T19:47:41.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:41 smithi116 ceph-mon[63578]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.116:0/1227733714"} : dispatch 2024-09-20T19:47:41.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:41.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:41 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:41 smithi118 ceph-mon[49197]: pgmap v334: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 25 KiB/s wr, 11 op/s 2024-09-20T19:47:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:41 smithi118 ceph-mon[49197]: from='mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497]' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.116:0/1227733714"} : dispatch 2024-09-20T19:47:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:41 smithi118 ceph-mon[49197]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.116:0/1227733714"} : dispatch 2024-09-20T19:47:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:41.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:41 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:42.801 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:42 smithi118 ceph-mon[49197]: Evicting (and blocklisting) client session 14552 (172.21.15.116:0/1227733714) 2024-09-20T19:47:42.801 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:42 smithi118 ceph-mon[49197]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.116:0/1227733714"}]': finished 2024-09-20T19:47:42.801 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:42 smithi118 ceph-mon[49197]: osdmap e164: 8 total, 8 up, 8 in 2024-09-20T19:47:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:42 smithi116 ceph-mon[63578]: Evicting (and blocklisting) client session 14552 (172.21.15.116:0/1227733714) 2024-09-20T19:47:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:42 smithi116 ceph-mon[63578]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.116:0/1227733714"}]': finished 2024-09-20T19:47:42.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:42 smithi116 ceph-mon[63578]: osdmap e164: 8 total, 8 up, 8 in 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: pgmap v336: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 307 B/s rd, 307 B/s wr, 0 op/s 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:47:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:43.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T19:47:43.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:47:43.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T19:47:43.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: pgmap v336: 97 pgs: 97 active+clean; 234 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 307 B/s rd, 307 B/s wr, 0 op/s 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T19:47:43.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T19:47:43.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi118.xrkvdr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T19:47:43.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:44.586 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:44 smithi118 ceph-mon[49197]: Upgrade: Updating nfs.foo.1.0.smithi118.xrkvdr (2/2) 2024-09-20T19:47:44.586 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:44 smithi118 ceph-mon[49197]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr 2024-09-20T19:47:44.586 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:44 smithi118 ceph-mon[49197]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T19:47:44.586 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:44 smithi118 ceph-mon[49197]: Rados config object exists: conf-nfs.foo 2024-09-20T19:47:44.586 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:44 smithi118 ceph-mon[49197]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr-rgw 2024-09-20T19:47:44.586 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:44 smithi118 ceph-mon[49197]: Bind address in nfs.foo.1.0.smithi118.xrkvdr's ganesha conf is defaulting to empty 2024-09-20T19:47:44.586 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:44 smithi118 ceph-mon[49197]: Deploying daemon nfs.foo.1.0.smithi118.xrkvdr on smithi118 2024-09-20T19:47:44.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:44 smithi116 ceph-mon[63578]: Upgrade: Updating nfs.foo.1.0.smithi118.xrkvdr (2/2) 2024-09-20T19:47:44.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:44 smithi116 ceph-mon[63578]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr 2024-09-20T19:47:44.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:44 smithi116 ceph-mon[63578]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T19:47:44.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:44 smithi116 ceph-mon[63578]: Rados config object exists: conf-nfs.foo 2024-09-20T19:47:44.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:44 smithi116 ceph-mon[63578]: Creating key for client.nfs.foo.1.0.smithi118.xrkvdr-rgw 2024-09-20T19:47:44.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:44 smithi116 ceph-mon[63578]: Bind address in nfs.foo.1.0.smithi118.xrkvdr's ganesha conf is defaulting to empty 2024-09-20T19:47:44.871 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:44 smithi116 ceph-mon[63578]: Deploying daemon nfs.foo.1.0.smithi118.xrkvdr on smithi118 2024-09-20T19:47:44.951 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (10m) 40s ago 19m 20.0M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (6m) 40s ago 19m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (6m) 14s ago 18m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (9m) 40s ago 18m 46.4M - dad864ee21e9 8a163a77164a 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (49s) 40s ago 13m 10.8M - 19.3.0-5127-ge8baca1e 82f5f45939a4 587a46003f50 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (17s) 14s ago 13m 14.4M - 19.3.0-5127-ge8baca1e 82f5f45939a4 c69eed77a2cf 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (9m) 40s ago 20m 502M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (8m) 14s ago 18m 451M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (8m) 40s ago 20m 60.6M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (7m) 14s ago 18m 52.5M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 starting - - - - 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (12m) 14s ago 12m 86.3M - 4.0 e1d6a67b021e c2587a74d4aa 2024-09-20T19:47:45.653 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (10m) 40s ago 18m 11.9M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (9m) 14s ago 18m 11.8M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (6m) 40s ago 17m 401M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (5m) 40s ago 17m 439M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (4m) 40s ago 16m 358M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (3m) 40s ago 16m 457M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (3m) 14s ago 15m 357M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (2m) 14s ago 15m 345M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (114s) 14s ago 15m 297M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 14b930edda7f 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (77s) 14s ago 14m 259M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 d60081632e22 2024-09-20T19:47:45.654 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (8m) 40s ago 18m 55.6M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:47:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:45 smithi118 ceph-mon[49197]: pgmap v337: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 921 B/s wr, 3 op/s 2024-09-20T19:47:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:45 smithi118 ceph-mon[49197]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:45.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:45 smithi118 ceph-mon[49197]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:45 smithi116 ceph-mon[63578]: pgmap v337: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 921 B/s wr, 3 op/s 2024-09-20T19:47:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:45 smithi116 ceph-mon[63578]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:45 smithi116 ceph-mon[63578]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 8 2024-09-20T19:47:46.042 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:46.043 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:47:46.043 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:47:46.043 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:47:46.043 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:47:46.043 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 14 2024-09-20T19:47:46.043 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:47:46.043 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:47:46.375 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "osd", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "mds", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "crash", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "mon" 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "16/23 daemons upgraded", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading nfs daemons", 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:47:46.376 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:47:46.688 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:46 smithi116 ceph-mon[63578]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:46.689 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:46 smithi116 ceph-mon[63578]: pgmap v338: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T19:47:46.689 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:46 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/4001967810' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:46.689 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:46 smithi116 ceph-mon[63578]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:47:46.777 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:47:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:46 smithi118 ceph-mon[49197]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:46 smithi118 ceph-mon[49197]: pgmap v338: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T19:47:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:46 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/4001967810' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:46.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:46 smithi118 ceph-mon[49197]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:47:47.592 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:47 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1950777069' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:47:47.622 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:47 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1950777069' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:47:47.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:47 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:47.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:47 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:47.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:47 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:47 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:47 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:47 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:48.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:48 smithi116 ceph-mon[63578]: pgmap v339: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.3 KiB/s wr, 7 op/s 2024-09-20T19:47:48.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:48 smithi118 ceph-mon[49197]: pgmap v339: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.3 KiB/s wr, 7 op/s 2024-09-20T19:47:50.781 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:50.782 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:50.782 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: Evicting (and blocklisting) client session 24309 (172.21.15.118:0/1616532918) 2024-09-20T19:47:50.782 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: from='mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497]' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/1616532918"} : dispatch 2024-09-20T19:47:50.782 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/1616532918"} : dispatch 2024-09-20T19:47:50.782 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: pgmap v340: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.4 KiB/s wr, 7 op/s 2024-09-20T19:47:50.782 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:50.782 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:50 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: Evicting (and blocklisting) client session 24309 (172.21.15.118:0/1616532918) 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: from='mds.? [v2:172.21.15.116:6834/2341653497,v1:172.21.15.116:6835/2341653497]' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/1616532918"} : dispatch 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/1616532918"} : dispatch 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: pgmap v340: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.4 KiB/s wr, 7 op/s 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:50 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:51.701 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:51 smithi118 ceph-mon[49197]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/1616532918"}]': finished 2024-09-20T19:47:51.701 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:51 smithi118 ceph-mon[49197]: osdmap e165: 8 total, 8 up, 8 in 2024-09-20T19:47:51.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:51 smithi116 ceph-mon[63578]: from='mds.? ' entity='mds.foofs.smithi116.xvfgaf' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.118:0/1616532918"}]': finished 2024-09-20T19:47:51.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:51 smithi116 ceph-mon[63578]: osdmap e165: 8 total, 8 up, 8 in 2024-09-20T19:47:52.691 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:52 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:52.692 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:52 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:52.692 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:52 smithi118 ceph-mon[49197]: pgmap v342: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-20T19:47:52.692 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:52 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:52.692 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:52 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:52.866 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:52 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:52.866 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:52 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:52.866 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:52 smithi116 ceph-mon[63578]: pgmap v342: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-20T19:47:52.866 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:52 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:52.866 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:52 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:54.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:54 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:54.157 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:54 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:54.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:54 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:54.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:54 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:55.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:55 smithi116 ceph-mon[63578]: pgmap v343: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-20T19:47:55.465 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:55 smithi118 ceph-mon[49197]: pgmap v343: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-20T19:47:57.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: pgmap v344: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T19:47:57.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:57.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:47:57.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.124 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all nfs 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi116.bfcxla"} : dispatch 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi116.bfcxla"}]': finished 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.xrkvdr"} : dispatch 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.xrkvdr"}]': finished 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: Upgrade: Setting container_image for all nvmeof 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.125 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:56 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: pgmap v344: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-20T19:47:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:47:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:47:57.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.216 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all nfs 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi116.bfcxla"} : dispatch 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi116.bfcxla"}]': finished 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.xrkvdr"} : dispatch 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi118.xrkvdr"}]': finished 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: Upgrade: Setting container_image for all nvmeof 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:57.217 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:56 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:47:58.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:57 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:58.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:57 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:47:59.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:58 smithi118 ceph-mon[49197]: Upgrade: Updating grafana.smithi116 2024-09-20T19:47:59.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:58 smithi118 ceph-mon[49197]: pgmap v345: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 511 B/s wr, 1 op/s 2024-09-20T19:47:59.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:47:58 smithi118 ceph-mon[49197]: Deploying daemon grafana.smithi116 on smithi116 2024-09-20T19:47:59.379 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:58 smithi116 ceph-mon[63578]: Upgrade: Updating grafana.smithi116 2024-09-20T19:47:59.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:58 smithi116 ceph-mon[63578]: pgmap v345: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 511 B/s wr, 1 op/s 2024-09-20T19:47:59.380 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:47:58 smithi116 ceph-mon[63578]: Deploying daemon grafana.smithi116 on smithi116 2024-09-20T19:48:01.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:00 smithi118 ceph-mon[49197]: pgmap v346: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-09-20T19:48:01.372 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:00 smithi116 ceph-mon[63578]: pgmap v346: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-09-20T19:48:03.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:02 smithi118 ceph-mon[49197]: pgmap v347: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 181 B/s rd, 272 B/s wr, 0 op/s 2024-09-20T19:48:03.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:02 smithi116 ceph-mon[63578]: pgmap v347: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 181 B/s rd, 272 B/s wr, 0 op/s 2024-09-20T19:48:05.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:04 smithi118 ceph-mon[49197]: pgmap v348: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-20T19:48:05.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:04 smithi116 ceph-mon[63578]: pgmap v348: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-09-20T19:48:07.175 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:06 smithi116 ceph-mon[63578]: pgmap v349: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-09-20T19:48:07.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:06 smithi118 ceph-mon[49197]: pgmap v349: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-09-20T19:48:09.043 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:08 smithi116 ceph-mon[63578]: pgmap v350: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:09.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:08 smithi118 ceph-mon[49197]: pgmap v350: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:11.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:11 smithi116 ceph-mon[63578]: pgmap v351: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:11.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:11.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:11.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:11 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:11 smithi118 ceph-mon[49197]: pgmap v351: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:11.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:11 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:13.517 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:13 smithi118 ceph-mon[49197]: pgmap v352: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:13.517 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:13 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:13.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:13 smithi116 ceph-mon[63578]: pgmap v352: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:13.616 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:13 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:15.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:15 smithi116 ceph-mon[63578]: pgmap v353: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:15.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:15 smithi118 ceph-mon[49197]: pgmap v353: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:16.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:16.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:16.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:16 smithi116 ceph-mon[63578]: pgmap v354: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:16.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:16.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:16 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:16.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:16.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:16.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:16 smithi118 ceph-mon[49197]: pgmap v354: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:16.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:16.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:16 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:17.136 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:48:17.795 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:48:17.795 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (10m) 2s ago 19m 20.0M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:48:17.795 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (7m) 2s ago 19m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:48:17.795 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (7m) 28s ago 19m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:48:17.795 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (8s) 2s ago 19m 40.9M - 10.4.8 5aad1d7cf004 8456bc6f023c 2024-09-20T19:48:17.795 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (81s) 2s ago 13m 32.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 587a46003f50 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (49s) 28s ago 13m 14.5M - 19.3.0-5127-ge8baca1e 82f5f45939a4 c69eed77a2cf 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (9m) 2s ago 20m 512M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (9m) 28s ago 19m 451M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (8m) 2s ago 20m 63.5M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (8m) 28s ago 18m 49.9M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (36s) 2s ago 13m 69.1M - 5.9 82f5f45939a4 ad393ca3e39d 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (31s) 28s ago 13m 17.5M - 5.9 82f5f45939a4 e438fe1e692f 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (10m) 2s ago 19m 11.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (10m) 28s ago 18m 11.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (6m) 2s ago 18m 416M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (6m) 2s ago 17m 456M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (5m) 2s ago 17m 389M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:48:17.796 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (4m) 2s ago 16m 483M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:48:17.797 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (3m) 28s ago 16m 359M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:48:17.797 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (3m) 28s ago 15m 345M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:48:17.797 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (2m) 28s ago 15m 296M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 14b930edda7f 2024-09-20T19:48:17.797 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (109s) 28s ago 15m 260M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 d60081632e22 2024-09-20T19:48:17.797 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (9m) 2s ago 18m 57.8M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 8 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:18.183 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:48:18.184 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:48:18.184 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:18.184 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:48:18.184 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 14 2024-09-20T19:48:18.184 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:48:18.184 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:48:18.526 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:48:18.526 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", 2024-09-20T19:48:18.526 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": true, 2024-09-20T19:48:18.526 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [ 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "osd", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "nfs", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "mds", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "crash", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "mon" 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": "18/23 daemons upgraded", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "Currently upgrading grafana daemons", 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:48:18.527 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:48:18.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:18 smithi118 ceph-mon[49197]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:18.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:18 smithi118 ceph-mon[49197]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:18.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:18 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/453340115' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:18.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:18 smithi116 ceph-mon[63578]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:18.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:18 smithi116 ceph-mon[63578]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:18.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:18 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/453340115' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:18.919 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:48:18.920 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:48:19.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:19 smithi118 ceph-mon[49197]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:19.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:19 smithi118 ceph-mon[49197]: pgmap v355: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:19.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:19 smithi118 ceph-mon[49197]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:19.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:19 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1285393575' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:48:19.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:19 smithi116 ceph-mon[63578]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:19.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:19 smithi116 ceph-mon[63578]: pgmap v355: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:19.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:19 smithi116 ceph-mon[63578]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:19.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:19 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1285393575' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:48:21.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:21 smithi118 ceph-mon[49197]: pgmap v356: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:21.741 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:21 smithi116 ceph-mon[63578]: pgmap v356: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:23.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: pgmap v357: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.716 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.717 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-20T19:48:23.718 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.719 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.720 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:23 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: pgmap v357: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:23.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-20T19:48:23.875 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-20T19:48:23.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.877 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:48:23.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:48:23.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:23.878 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:23 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:24.788 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: Failed to get Dashboard config for Grafana: dashboard get-grafana-api-url failed: command not known retval: -22 2024-09-20T19:48:24.788 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: Upgrade: Finalizing container_image settings 2024-09-20T19:48:24.788 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: Upgrade: Complete! 2024-09-20T19:48:24.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:24.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:48:24.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:48:24.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:24.789 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: Failed to get Dashboard config for Grafana: dashboard get-grafana-api-url failed: command not known retval: -22 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: Upgrade: Finalizing container_image settings 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: Upgrade: Complete! 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:25.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:25 smithi116 ceph-mon[63578]: pgmap v358: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:25.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:25 smithi118 ceph-mon[49197]: pgmap v358: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:27.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:27 smithi116 ceph-mon[63578]: pgmap v359: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:27.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:27 smithi118 ceph-mon[49197]: pgmap v359: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:28.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:28.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:48:29.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:29 smithi116 ceph-mon[63578]: pgmap v360: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:29.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:29 smithi118 ceph-mon[49197]: pgmap v360: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:31.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:31 smithi116 ceph-mon[63578]: pgmap v361: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:31 smithi118 ceph-mon[49197]: pgmap v361: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:33.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:33 smithi116 ceph-mon[63578]: pgmap v362: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:33.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:33 smithi118 ceph-mon[49197]: pgmap v362: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:35.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:35 smithi116 ceph-mon[63578]: pgmap v363: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:35.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:35 smithi118 ceph-mon[49197]: pgmap v363: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:37.874 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:37 smithi116 ceph-mon[63578]: pgmap v364: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:37.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:37 smithi118 ceph-mon[49197]: pgmap v364: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:39.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:39 smithi116 ceph-mon[63578]: pgmap v365: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:39 smithi118 ceph-mon[49197]: pgmap v365: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:41.876 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:41 smithi116 ceph-mon[63578]: pgmap v366: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:41.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:41 smithi118 ceph-mon[49197]: pgmap v366: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:43.767 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:43 smithi118 ceph-mon[49197]: pgmap v367: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:43.767 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:43 smithi116 ceph-mon[63578]: pgmap v367: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:45 smithi116 ceph-mon[63578]: pgmap v368: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:45.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:45 smithi118 ceph-mon[49197]: pgmap v368: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:47 smithi116 ceph-mon[63578]: pgmap v369: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:47 smithi118 ceph-mon[49197]: pgmap v369: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:49.554 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:49 smithi116 ceph-mon[63578]: pgmap v370: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:49.959 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph orch ps' 2024-09-20T19:48:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:49 smithi118 ceph-mon[49197]: pgmap v370: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:50.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:50 smithi116 ceph-mon[63578]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:50.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:50 smithi118 ceph-mon[49197]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:51.725 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:51 smithi116 ceph-mon[63578]: pgmap v371: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:51.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:51 smithi118 ceph-mon[49197]: pgmap v371: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:52.055 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:48:52.055 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (11m) 36s ago 20m 20.0M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:48:52.055 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (8m) 36s ago 20m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:48:52.055 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (7m) 62s ago 19m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:48:52.055 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (42s) 36s ago 19m 40.9M - 10.4.8 5aad1d7cf004 8456bc6f023c 2024-09-20T19:48:52.055 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (116s) 36s ago 14m 32.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 587a46003f50 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (84s) 62s ago 14m 14.5M - 19.3.0-5127-ge8baca1e 82f5f45939a4 c69eed77a2cf 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (10m) 36s ago 21m 512M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (9m) 62s ago 19m 451M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (9m) 36s ago 21m 63.5M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (8m) 62s ago 19m 49.9M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (71s) 36s ago 13m 69.1M - 5.9 82f5f45939a4 ad393ca3e39d 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (65s) 62s ago 13m 17.5M - 5.9 82f5f45939a4 e438fe1e692f 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (11m) 36s ago 19m 11.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (10m) 62s ago 19m 11.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (7m) 36s ago 18m 416M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (6m) 36s ago 18m 456M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (5m) 36s ago 17m 389M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (5m) 36s ago 17m 483M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:48:52.056 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (4m) 62s ago 16m 359M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:48:52.057 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (3m) 62s ago 16m 345M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:48:52.057 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (3m) 62s ago 16m 296M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 14b930edda7f 2024-09-20T19:48:52.057 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (2m) 62s ago 15m 260M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 d60081632e22 2024-09-20T19:48:52.057 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (9m) 36s ago 19m 57.8M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:48:52.692 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph versions' 2024-09-20T19:48:53.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:53 smithi116 ceph-mon[63578]: pgmap v372: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:53.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:53 smithi116 ceph-mon[63578]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:53.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:53 smithi118 ceph-mon[49197]: pgmap v372: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:53.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:53 smithi118 ceph-mon[49197]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:48:54.751 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 8 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 14 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:48:54.752 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:48:55.476 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-20T19:48:55.768 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:55 smithi116 ceph-mon[63578]: pgmap v373: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:55.768 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:55 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2775692577' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:55 smithi118 ceph-mon[49197]: pgmap v373: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:48:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:55 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2775692577' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:48:57.204 INFO:teuthology.orchestra.run.smithi116.stdout:wait for servicemap items w/ changing names to refresh 2024-09-20T19:48:57.741 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:57 smithi116 ceph-mon[63578]: pgmap v374: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:57.765 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'sleep 60' 2024-09-20T19:48:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:57 smithi118 ceph-mon[49197]: pgmap v374: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:58.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:48:59.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:48:59 smithi116 ceph-mon[63578]: pgmap v375: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:48:59.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:48:59 smithi118 ceph-mon[49197]: pgmap v375: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:01.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:01 smithi116 ceph-mon[63578]: pgmap v376: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:01.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:01 smithi118 ceph-mon[49197]: pgmap v376: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:03 smithi116 ceph-mon[63578]: pgmap v377: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:03.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:03 smithi118 ceph-mon[49197]: pgmap v377: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:05.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:05 smithi116 ceph-mon[63578]: pgmap v378: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:05.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:05 smithi118 ceph-mon[49197]: pgmap v378: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:07.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:07 smithi116 ceph-mon[63578]: pgmap v379: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:07.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:07 smithi118 ceph-mon[49197]: pgmap v379: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:08.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:08 smithi116 ceph-mon[63578]: pgmap v380: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:08 smithi118 ceph-mon[49197]: pgmap v380: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:11.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:10 smithi118 ceph-mon[49197]: pgmap v381: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:11.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:10 smithi116 ceph-mon[63578]: pgmap v381: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:13.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:12 smithi118 ceph-mon[49197]: pgmap v382: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:13.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:13.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:12 smithi116 ceph-mon[63578]: pgmap v382: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:13.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:15.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:14 smithi118 ceph-mon[49197]: pgmap v383: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:15.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:14 smithi116 ceph-mon[63578]: pgmap v383: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:17.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:16 smithi118 ceph-mon[49197]: pgmap v384: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:17.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:16 smithi116 ceph-mon[63578]: pgmap v384: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:18 smithi118 ceph-mon[49197]: pgmap v385: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:19.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:18 smithi116 ceph-mon[63578]: pgmap v385: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:21.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:20 smithi118 ceph-mon[49197]: pgmap v386: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:21.373 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:20 smithi116 ceph-mon[63578]: pgmap v386: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:23.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:23 smithi118 ceph-mon[49197]: pgmap v387: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:23.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:23 smithi116 ceph-mon[63578]: pgmap v387: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:49:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:49:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:49:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:24.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:49:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:49:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:49:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:24.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:49:25.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:25 smithi116 ceph-mon[63578]: pgmap v388: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:25.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:25 smithi118 ceph-mon[49197]: pgmap v388: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:27.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:27 smithi116 ceph-mon[63578]: pgmap v389: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:27.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:27 smithi118 ceph-mon[49197]: pgmap v389: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:28.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:28.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:29.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:29 smithi116 ceph-mon[63578]: pgmap v390: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:29.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:29 smithi118 ceph-mon[49197]: pgmap v390: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:31.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:31 smithi116 ceph-mon[63578]: pgmap v391: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:31.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:31 smithi118 ceph-mon[49197]: pgmap v391: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:33.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:33 smithi116 ceph-mon[63578]: pgmap v392: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:33.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:33 smithi118 ceph-mon[49197]: pgmap v392: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:35.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:35 smithi116 ceph-mon[63578]: pgmap v393: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:35.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:35 smithi118 ceph-mon[49197]: pgmap v393: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:37.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:37 smithi116 ceph-mon[63578]: pgmap v394: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:37.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:37 smithi118 ceph-mon[49197]: pgmap v394: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:39.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:39 smithi116 ceph-mon[63578]: pgmap v395: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:39.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:39 smithi118 ceph-mon[49197]: pgmap v395: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:41.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:41 smithi116 ceph-mon[63578]: pgmap v396: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:41.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:41 smithi118 ceph-mon[49197]: pgmap v396: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:43.769 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:43 smithi118 ceph-mon[49197]: pgmap v397: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:43.769 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:43 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:43 smithi116 ceph-mon[63578]: pgmap v397: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:43.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:43 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:45.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:45 smithi116 ceph-mon[63578]: pgmap v398: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:45.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:45 smithi118 ceph-mon[49197]: pgmap v398: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:47.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:47 smithi116 ceph-mon[63578]: pgmap v399: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:47.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:47 smithi118 ceph-mon[49197]: pgmap v399: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:49.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:49 smithi116 ceph-mon[63578]: pgmap v400: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:49.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:49 smithi118 ceph-mon[49197]: pgmap v400: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:51.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:51 smithi116 ceph-mon[63578]: pgmap v401: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:51.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:51 smithi118 ceph-mon[49197]: pgmap v401: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:53.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:53 smithi116 ceph-mon[63578]: pgmap v402: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:53.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:53 smithi118 ceph-mon[49197]: pgmap v402: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:55.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:55 smithi116 ceph-mon[63578]: pgmap v403: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:55.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:55 smithi118 ceph-mon[49197]: pgmap v403: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:49:57.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:57 smithi116 ceph-mon[63578]: pgmap v404: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:57.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:57 smithi118 ceph-mon[49197]: pgmap v404: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:58.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:58 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:58.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:58 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:49:59.711 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:49:59 smithi116 ceph-mon[63578]: pgmap v405: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:49:59.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:49:59 smithi118 ceph-mon[49197]: pgmap v405: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:00.117 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph orch ps' 2024-09-20T19:50:00.681 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: Health detail: HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:50:00.681 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: [WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:50:00.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:50:00.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: [WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:50:00.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:50:00.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:50:00.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:50:00.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:50:00.682 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:00 smithi116 ceph-mon[63578]: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:50:00.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: Health detail: HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:50:00.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: [WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:50:00.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:50:00.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: [WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:50:00.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:50:00.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:50:00.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:50:00.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:50:00.966 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:00 smithi118 ceph-mon[49197]: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:50:01.575 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:01 smithi116 ceph-mon[63578]: pgmap v406: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:01.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:01 smithi118 ceph-mon[49197]: pgmap v406: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:02.016 INFO:teuthology.orchestra.run.smithi116.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T19:50:02.016 INFO:teuthology.orchestra.run.smithi116.stdout:alertmanager.smithi116 smithi116 *:9093,9094 running (12m) 106s ago 21m 20.0M - 0.27.0 11f11916f8cd 6fbe443aa84d 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi116 smithi116 running (9m) 106s ago 21m 6920k - 19.3.0-5127-ge8baca1e 82f5f45939a4 f11463133f5f 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:crash.smithi118 smithi118 running (9m) 2m ago 20m 6937k - 19.3.0-5127-ge8baca1e 82f5f45939a4 3a2e237fef82 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:grafana.smithi116 smithi116 *:3000 running (112s) 106s ago 21m 40.9M - 10.4.8 5aad1d7cf004 8456bc6f023c 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi116.xvfgaf smithi116 running (3m) 106s ago 15m 32.6M - 19.3.0-5127-ge8baca1e 82f5f45939a4 587a46003f50 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:mds.foofs.smithi118.zumcyi smithi118 running (2m) 2m ago 15m 14.5M - 19.3.0-5127-ge8baca1e 82f5f45939a4 c69eed77a2cf 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi116.poxcrq smithi116 *:9283,8765 running (11m) 106s ago 22m 512M - 19.3.0-5127-ge8baca1e 82f5f45939a4 92501d20128c 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:mgr.smithi118.rmiazv smithi118 *:9283,8765 running (10m) 2m ago 20m 451M - 19.3.0-5127-ge8baca1e 82f5f45939a4 41d7769cf092 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi116 smithi116 running (10m) 106s ago 22m 63.5M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 f2358e4c35ef 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:mon.smithi118 smithi118 running (10m) 2m ago 20m 49.9M 2048M 19.3.0-5127-ge8baca1e 82f5f45939a4 7103ca9965a4 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.0.0.smithi116.bfcxla smithi116 *:2049 running (2m) 106s ago 15m 69.1M - 5.9 82f5f45939a4 ad393ca3e39d 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:nfs.foo.1.0.smithi118.xrkvdr smithi118 *:2049 running (2m) 2m ago 14m 17.5M - 5.9 82f5f45939a4 e438fe1e692f 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi116 smithi116 *:9100 running (12m) 106s ago 21m 11.2M - 1.7.0 72c9c2088986 44c56a8d0be1 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:node-exporter.smithi118 smithi118 *:9100 running (11m) 2m ago 20m 11.2M - 1.7.0 72c9c2088986 831278d48a00 2024-09-20T19:50:02.017 INFO:teuthology.orchestra.run.smithi116.stdout:osd.0 smithi116 running (8m) 106s ago 19m 416M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 cbf89945fd9e 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:osd.1 smithi116 running (7m) 106s ago 19m 456M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 d48d1daf3d38 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:osd.2 smithi116 running (7m) 106s ago 18m 389M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e0b40360570b 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:osd.3 smithi116 running (6m) 106s ago 18m 483M 1936M 19.3.0-5127-ge8baca1e 82f5f45939a4 e3b204ab7794 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:osd.4 smithi118 running (5m) 2m ago 18m 359M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 4ae34596cfea 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:osd.5 smithi118 running (4m) 2m ago 17m 345M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 e6a039aa388f 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:osd.6 smithi118 running (4m) 2m ago 17m 296M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 14b930edda7f 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:osd.7 smithi118 running (3m) 2m ago 16m 260M 2704M 19.3.0-5127-ge8baca1e 82f5f45939a4 d60081632e22 2024-09-20T19:50:02.018 INFO:teuthology.orchestra.run.smithi116.stdout:prometheus.smithi116 smithi116 *:9095 running (10m) 106s ago 20m 57.8M - 2.51.0 1d3b7f56885b 9c15607daa4c 2024-09-20T19:50:02.675 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph orch upgrade status' 2024-09-20T19:50:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:03 smithi116 ceph-mon[63578]: pgmap v407: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:03.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:03 smithi116 ceph-mon[63578]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:03.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:03 smithi118 ceph-mon[49197]: pgmap v407: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:03.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:03 smithi118 ceph-mon[49197]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:04.669 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:50:04.669 INFO:teuthology.orchestra.run.smithi116.stdout: "target_image": null, 2024-09-20T19:50:04.669 INFO:teuthology.orchestra.run.smithi116.stdout: "in_progress": false, 2024-09-20T19:50:04.670 INFO:teuthology.orchestra.run.smithi116.stdout: "which": "", 2024-09-20T19:50:04.670 INFO:teuthology.orchestra.run.smithi116.stdout: "services_complete": [], 2024-09-20T19:50:04.670 INFO:teuthology.orchestra.run.smithi116.stdout: "progress": null, 2024-09-20T19:50:04.670 INFO:teuthology.orchestra.run.smithi116.stdout: "message": "", 2024-09-20T19:50:04.670 INFO:teuthology.orchestra.run.smithi116.stdout: "is_paused": false 2024-09-20T19:50:04.670 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:50:05.528 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph health detail' 2024-09-20T19:50:05.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:05 smithi116 ceph-mon[63578]: pgmap v408: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:50:05.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:05 smithi118 ceph-mon[49197]: pgmap v408: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:50:06.832 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:06 smithi116 ceph-mon[63578]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:06.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:06 smithi118 ceph-mon[49197]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout:HEALTH_WARN Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251); 5 mgr modules have recently crashed 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] MGR_MODULE_DEPENDENCY: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout: Module 'dashboard' has failed dependency: EOL while scanning string literal (rgw_client.py, line 251) 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout:[WRN] RECENT_MGR_MODULE_CRASH: 5 mgr modules have recently crashed 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:38:27.633960Z 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi116.poxcrq on host smithi116 at 2024-09-20T19:39:41.831790Z 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:36:46.874447Z 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:19.182028Z 2024-09-20T19:50:07.616 INFO:teuthology.orchestra.run.smithi116.stdout: mgr module dashboard crashed in daemon mgr.smithi118.rmiazv on host smithi118 at 2024-09-20T19:39:41.743379Z 2024-09-20T19:50:07.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:07 smithi116 ceph-mon[63578]: pgmap v409: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:07.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:07 smithi118 ceph-mon[49197]: pgmap v409: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:08.198 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph versions' 2024-09-20T19:50:08.802 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:08 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1801019691' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:50:08.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:08 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1801019691' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T19:50:09.834 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:09 smithi116 ceph-mon[63578]: pgmap v410: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:09.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:09 smithi118 ceph-mon[49197]: pgmap v410: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:10.216 INFO:teuthology.orchestra.run.smithi116.stdout:{ 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "mon": { 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "mgr": { 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "osd": { 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 8 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "mds": { 2024-09-20T19:50:10.217 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 2 2024-09-20T19:50:10.218 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:50:10.218 INFO:teuthology.orchestra.run.smithi116.stdout: "overall": { 2024-09-20T19:50:10.218 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)": 14 2024-09-20T19:50:10.218 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:50:10.218 INFO:teuthology.orchestra.run.smithi116.stdout:} 2024-09-20T19:50:10.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:10 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/3280443425' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:50:10.920 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-20T19:50:10.943 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:10 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/3280443425' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:50:11.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:11 smithi116 ceph-mon[63578]: pgmap v411: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:11.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:11 smithi118 ceph-mon[49197]: pgmap v411: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:12.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:12 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:50:12.955 INFO:teuthology.orchestra.run.smithi116.stdout:true 2024-09-20T19:50:12.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:12 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:50:13.561 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -e sha1=e8baca1eac6c3c9c88bf2ea4493e448bd54936a7 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-20T19:50:13.860 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:13 smithi116 ceph-mon[63578]: pgmap v412: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:13.860 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:13 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/2263124365' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:50:13.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:13 smithi118 ceph-mon[49197]: pgmap v412: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:13.866 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:13 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/2263124365' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:50:14.855 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:14 smithi116 ceph-mon[63578]: pgmap v413: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:50:14.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:14 smithi118 ceph-mon[49197]: pgmap v413: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:50:15.470 INFO:teuthology.orchestra.run.smithi116.stdout: "ceph version 19.3.0-5127-ge8baca1e (e8baca1eac6c3c9c88bf2ea4493e448bd54936a7) squid (dev)" 2024-09-20T19:50:15.873 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:15 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/1871598830' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:50:15.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:15 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/1871598830' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T19:50:16.007 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-20T19:50:16.008 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-20T19:50:16.008 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T19:50:16.872 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:16 smithi116 ceph-mon[63578]: pgmap v414: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:16.965 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:16 smithi118 ceph-mon[49197]: pgmap v414: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:18.033 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-09-20T19:50:18.033 INFO:teuthology.orchestra.run.smithi116.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T19:28:02.827016Z", "last_refresh": "2024-09-20T19:48:15.644786Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T19:27:56.093943Z", "last_refresh": "2024-09-20T19:47:49.546053Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-20T19:27:59.549128Z", "last_refresh": "2024-09-20T19:48:15.644932Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T19:34:35.225490Z", "last_refresh": "2024-09-20T19:47:49.546613Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T19:27:54.712090Z", "last_refresh": "2024-09-20T19:47:49.546162Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi116:172.21.15.116=smithi116", "smithi118:172.21.15.118=smithi118"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T19:28:45.778166Z", "last_refresh": "2024-09-20T19:47:49.546231Z", "running": 2, "size": 2}}, {"events": ["2024-09-20T19:49:24.432868Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-20T19:34:53.614986Z", "last_refresh": "2024-09-20T19:47:49.546677Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T19:28:01.506994Z", "last_refresh": "2024-09-20T19:47:49.546296Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d", "last_refresh": "2024-09-20T19:47:49.546361Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T19:27:57.573016Z", "last_refresh": "2024-09-20T19:48:15.645064Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T19:50:18.674 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-20T19:50:18.674 DEBUG:teuthology.parallel:result is None 2024-09-20T19:50:18.674 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T19:50:18.684 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:50:18.684 DEBUG:teuthology.orchestra.run.smithi116:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-20T19:50:18.709 INFO:teuthology.orchestra.run.smithi116.stderr:+ umount /mnt/foo 2024-09-20T19:50:18.729 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T19:50:18.738 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:50:18.738 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-20T19:50:19.184 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:18 smithi116 ceph-mon[63578]: pgmap v415: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:19.184 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:18 smithi116 ceph-mon[63578]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:50:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:18 smithi118 ceph-mon[49197]: pgmap v415: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:19.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:18 smithi118 ceph-mon[49197]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T19:50:20.684 INFO:teuthology.orchestra.run.smithi116.stdout: "foo" 2024-09-20T19:50:21.088 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:20 smithi116 ceph-mon[63578]: pgmap v416: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:21.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:20 smithi118 ceph-mon[49197]: pgmap v416: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:21.290 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-20T19:50:22.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:21 smithi118 ceph-mon[49197]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:22.238 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:21 smithi116 ceph-mon[63578]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:23.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:22 smithi116 ceph-mon[63578]: pgmap v417: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:23.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:22 smithi118 ceph-mon[49197]: pgmap v417: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:23.220 INFO:teuthology.orchestra.run.smithi116.stdout:[ 2024-09-20T19:50:23.220 INFO:teuthology.orchestra.run.smithi116.stdout: { 2024-09-20T19:50:23.220 INFO:teuthology.orchestra.run.smithi116.stdout: "access_type": "RW", 2024-09-20T19:50:23.220 INFO:teuthology.orchestra.run.smithi116.stdout: "clients": [], 2024-09-20T19:50:23.220 INFO:teuthology.orchestra.run.smithi116.stdout: "cluster_id": "foo", 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "export_id": 1, 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "fsal": { 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "fs_name": "foofs", 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "name": "CEPH", 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "user_id": "nfs.foo.1" 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: }, 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "path": "/", 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "protocols": [ 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: 4 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: ], 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "pseudo": "/fake", 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "security_label": true, 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "squash": "none", 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "transports": [ 2024-09-20T19:50:23.221 INFO:teuthology.orchestra.run.smithi116.stdout: "TCP" 2024-09-20T19:50:23.222 INFO:teuthology.orchestra.run.smithi116.stdout: ] 2024-09-20T19:50:23.222 INFO:teuthology.orchestra.run.smithi116.stdout: } 2024-09-20T19:50:23.222 INFO:teuthology.orchestra.run.smithi116.stdout:] 2024-09-20T19:50:23.844 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'rados -p .nfs --all ls -' 2024-09-20T19:50:24.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:23 smithi116 ceph-mon[63578]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:24.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:23 smithi118 ceph-mon[49197]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T19:50:25.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:24 smithi118 ceph-mon[49197]: pgmap v418: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:50:25.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:24 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:50:25.221 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:24 smithi116 ceph-mon[63578]: pgmap v418: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T19:50:25.222 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:24 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T19:50:25.599 INFO:teuthology.orchestra.run.smithi116.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-20T19:50:25.599 INFO:teuthology.orchestra.run.smithi116.stdout:foo grace 2024-09-20T19:50:25.599 INFO:teuthology.orchestra.run.smithi116.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-20T19:50:25.599 INFO:teuthology.orchestra.run.smithi116.stdout:foo conf-nfs.foo 2024-09-20T19:50:25.599 INFO:teuthology.orchestra.run.smithi116.stdout:foo export-1 2024-09-20T19:50:26.184 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T19:50:26.193 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:50:26.193 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 -- bash -c 'set -ex 2024-09-20T19:50:26.193 DEBUG:teuthology.orchestra.run.smithi116:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-20T19:50:26.193 DEBUG:teuthology.orchestra.run.smithi116:> ' 2024-09-20T19:50:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:25 smithi118 ceph-mon[49197]: mgrmap e44: smithi116.poxcrq(active, since 10m), standbys: smithi118.rmiazv 2024-09-20T19:50:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:50:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:50:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:50:26.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:25 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:50:26.217 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:25 smithi116 ceph-mon[63578]: mgrmap e44: smithi116.poxcrq(active, since 10m), standbys: smithi118.rmiazv 2024-09-20T19:50:26.217 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T19:50:26.217 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T19:50:26.217 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:50:26.217 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:25 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' 2024-09-20T19:50:27.452 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:27 smithi116 ceph-mon[63578]: pgmap v419: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:27.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:27 smithi118 ceph-mon[49197]: pgmap v419: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T19:50:27.939 INFO:teuthology.orchestra.run.smithi116.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-20T19:50:28.190 INFO:teuthology.orchestra.run.smithi116.stderr:+ [[ 7 -gt 2 ]] 2024-09-20T19:50:28.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:28 smithi116 ceph-mon[63578]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:50:28.527 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:28 smithi116 ceph-mon[63578]: from='client.? 172.21.15.116:0/802079589' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-20T19:50:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:28 smithi118 ceph-mon[49197]: from='mgr.34100 172.21.15.116:0/1943738679' entity='mgr.smithi116.poxcrq' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T19:50:28.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:28 smithi118 ceph-mon[49197]: from='client.? 172.21.15.116:0/802079589' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-20T19:50:28.850 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-20T19:50:28.860 INFO:tasks.cephadm:Teardown begin 2024-09-20T19:50:28.860 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T19:50:28.917 DEBUG:teuthology.orchestra.run.smithi118:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T19:50:28.950 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-20T19:50:28.950 DEBUG:teuthology.orchestra.run.smithi116:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-20T19:50:28.973 DEBUG:teuthology.orchestra.run.smithi118:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-20T19:50:29.009 INFO:tasks.cephadm:Stopping all daemons... 2024-09-20T19:50:29.009 INFO:tasks.cephadm.mon.smithi116:Stopping mon.smithi116... 2024-09-20T19:50:29.009 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116 2024-09-20T19:50:29.123 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 systemd[1]: Stopping Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:50:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 ceph-mon[63578]: pgmap v420: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T19:50:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[63574]: 2024-09-20T19:50:29.304+0000 7fdda4547640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi116 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:50:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116[63574]: 2024-09-20T19:50:29.304+0000 7fdda4547640 -1 mon.smithi116@0(leader) e3 *** Got Signal Terminated *** 2024-09-20T19:50:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 podman[93345]: 2024-09-20 19:50:29.34546616 +0000 UTC m=+0.150878664 container stop f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-20T19:50:29.623 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 podman[93345]: 2024-09-20 19:50:29.372949109 +0000 UTC m=+0.178361615 container died f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:50:29.715 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:29 smithi118 ceph-mon[49197]: pgmap v420: 97 pgs: 97 active+clean; 234 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T19:50:29.932 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 podman[93345]: 2024-09-20 19:50:29.746366329 +0000 UTC m=+0.551778830 container cleanup f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-20T19:50:29.932 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 bash[93345]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116 2024-09-20T19:50:30.171 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service' 2024-09-20T19:50:30.192 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:29 smithi116 podman[93357]: 2024-09-20 19:50:29.93102144 +0000 UTC m=+0.579736892 container remove f2358e4c35efc2a33045189baa4bd4e1cdc815a001d36cf62a440113003dc383 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi116, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-20T19:50:30.193 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:30 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service: Deactivated successfully. 2024-09-20T19:50:30.193 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:30 smithi116 systemd[1]: Stopped Ceph mon.smithi116 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:50:30.193 INFO:journalctl@ceph.mon.smithi116.smithi116.stdout:Sep 20 19:50:30 smithi116 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi116.service: Consumed 13.244s CPU time. 2024-09-20T19:50:30.226 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:50:30.226 INFO:tasks.cephadm.mon.smithi116:Stopped mon.smithi116 2024-09-20T19:50:30.226 INFO:tasks.cephadm.mon.smithi118:Stopping mon.smithi118... 2024-09-20T19:50:30.226 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi118 2024-09-20T19:50:30.536 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:30 smithi118 systemd[1]: Stopping Ceph mon.smithi118 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:50:30.870 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118[49193]: 2024-09-20T19:50:30.533+0000 7f40d27bd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi118 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:50:30.870 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118[49193]: 2024-09-20T19:50:30.533+0000 7f40d27bd640 -1 mon.smithi118@1(peon) e3 *** Got Signal Terminated *** 2024-09-20T19:50:30.870 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:30 smithi118 podman[68013]: 2024-09-20 19:50:30.565165795 +0000 UTC m=+0.137519720 container died 7103ca9965a41f267220b67dcddbf3350f6496ec91d5203110baccf1fd501ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-20T19:50:31.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:31 smithi118 podman[68013]: 2024-09-20 19:50:31.051161342 +0000 UTC m=+0.623515299 container cleanup 7103ca9965a41f267220b67dcddbf3350f6496ec91d5203110baccf1fd501ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:50:31.215 INFO:journalctl@ceph.mon.smithi118.smithi118.stdout:Sep 20 19:50:31 smithi118 bash[68013]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-mon-smithi118 2024-09-20T19:50:31.496 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@mon.smithi118.service' 2024-09-20T19:50:31.553 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:50:31.554 INFO:tasks.cephadm.mon.smithi118:Stopped mon.smithi118 2024-09-20T19:50:31.554 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-20T19:50:31.554 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.0 2024-09-20T19:50:31.873 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:31 smithi116 systemd[1]: Stopping Ceph osd.0 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:50:32.373 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:31 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[68342]: 2024-09-20T19:50:31.875+0000 7f1bce515640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:50:32.373 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:31 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[68342]: 2024-09-20T19:50:31.875+0000 7f1bce515640 -1 osd.0 165 *** Got signal Terminated *** 2024-09-20T19:50:32.373 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:31 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0[68342]: 2024-09-20T19:50:31.875+0000 7f1bce515640 -1 osd.0 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:50:37.745 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:37 smithi116 podman[93480]: 2024-09-20 19:50:37.430667121 +0000 UTC m=+5.685046687 container died cbf89945fd9e87414fd895d1d513bf801f66ac5ea9f5946e49b8c4f31fbb80e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:50:38.107 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:37 smithi116 podman[93480]: 2024-09-20 19:50:37.825741204 +0000 UTC m=+6.080120769 container cleanup cbf89945fd9e87414fd895d1d513bf801f66ac5ea9f5946e49b8c4f31fbb80e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T19:50:38.108 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:37 smithi116 bash[93480]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0 2024-09-20T19:50:38.373 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:38 smithi116 podman[93513]: 2024-09-20 19:50:38.106697018 +0000 UTC m=+0.667424166 container remove cbf89945fd9e87414fd895d1d513bf801f66ac5ea9f5946e49b8c4f31fbb80e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-09-20T19:50:38.707 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:38 smithi116 podman[93602]: 2024-09-20 19:50:38.348946669 +0000 UTC m=+0.016022487 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:50:38.707 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:38 smithi116 podman[93602]: 2024-09-20 19:50:38.453617671 +0000 UTC m=+0.120693478 container create 141ac791e00f7f4d365fca030cb996b10748aab215a6e5284c51220b109d2d80 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:50:38.707 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:38 smithi116 podman[93602]: 2024-09-20 19:50:38.706361016 +0000 UTC m=+0.373436830 container init 141ac791e00f7f4d365fca030cb996b10748aab215a6e5284c51220b109d2d80 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-20T19:50:39.123 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:38 smithi116 podman[93602]: 2024-09-20 19:50:38.709987029 +0000 UTC m=+0.377062841 container start 141ac791e00f7f4d365fca030cb996b10748aab215a6e5284c51220b109d2d80 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:50:39.123 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:38 smithi116 podman[93602]: 2024-09-20 19:50:38.798098664 +0000 UTC m=+0.465174469 container attach 141ac791e00f7f4d365fca030cb996b10748aab215a6e5284c51220b109d2d80 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:50:39.123 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:38 smithi116 podman[93602]: 2024-09-20 19:50:38.894955355 +0000 UTC m=+0.562031162 container died 141ac791e00f7f4d365fca030cb996b10748aab215a6e5284c51220b109d2d80 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-20T19:50:39.431 INFO:journalctl@ceph.osd.0.smithi116.stdout:Sep 20 19:50:39 smithi116 podman[93625]: 2024-09-20 19:50:39.426714547 +0000 UTC m=+0.525822758 container remove 141ac791e00f7f4d365fca030cb996b10748aab215a6e5284c51220b109d2d80 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-0-deactivate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T19:50:39.452 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.0.service' 2024-09-20T19:50:39.492 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:50:39.492 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-20T19:50:39.493 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-20T19:50:39.493 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.1 2024-09-20T19:50:39.832 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:39 smithi116 systemd[1]: Stopping Ceph osd.1 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:50:40.123 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:39 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[70713]: 2024-09-20T19:50:39.830+0000 7f5e140b0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:50:40.123 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:39 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[70713]: 2024-09-20T19:50:39.830+0000 7f5e140b0640 -1 osd.1 165 *** Got signal Terminated *** 2024-09-20T19:50:40.123 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:39 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1[70713]: 2024-09-20T19:50:39.830+0000 7f5e140b0640 -1 osd.1 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:50:45.623 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:45 smithi116 podman[93707]: 2024-09-20 19:50:45.370074161 +0000 UTC m=+5.653473309 container died d48d1daf3d38232800ec75a15c4f2b53d5f0a45de1f7043b8a141faf28e23bc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True) 2024-09-20T19:50:46.040 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:45 smithi116 podman[93707]: 2024-09-20 19:50:45.721247639 +0000 UTC m=+6.004646780 container cleanup d48d1daf3d38232800ec75a15c4f2b53d5f0a45de1f7043b8a141faf28e23bc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True) 2024-09-20T19:50:46.040 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:45 smithi116 bash[93707]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1 2024-09-20T19:50:46.373 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:46 smithi116 podman[93725]: 2024-09-20 19:50:46.039213347 +0000 UTC m=+0.667059473 container remove d48d1daf3d38232800ec75a15c4f2b53d5f0a45de1f7043b8a141faf28e23bc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:50:46.702 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:46 smithi116 podman[93811]: 2024-09-20 19:50:46.295804987 +0000 UTC m=+0.015803164 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:50:46.702 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:46 smithi116 podman[93811]: 2024-09-20 19:50:46.431867152 +0000 UTC m=+0.151865325 container create dc5fdcd451343900dc492e57b181d9c2b213029dd126d9f2980506f5a0455c4a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2) 2024-09-20T19:50:46.702 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:46 smithi116 podman[93811]: 2024-09-20 19:50:46.701202865 +0000 UTC m=+0.421201044 container init dc5fdcd451343900dc492e57b181d9c2b213029dd126d9f2980506f5a0455c4a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-20T19:50:47.123 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:46 smithi116 podman[93811]: 2024-09-20 19:50:46.704877646 +0000 UTC m=+0.424875826 container start dc5fdcd451343900dc492e57b181d9c2b213029dd126d9f2980506f5a0455c4a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:50:47.123 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:46 smithi116 podman[93811]: 2024-09-20 19:50:46.809823023 +0000 UTC m=+0.529821205 container attach dc5fdcd451343900dc492e57b181d9c2b213029dd126d9f2980506f5a0455c4a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:50:47.123 INFO:journalctl@ceph.osd.1.smithi116.stdout:Sep 20 19:50:46 smithi116 podman[93811]: 2024-09-20 19:50:46.903346725 +0000 UTC m=+0.623344898 container died dc5fdcd451343900dc492e57b181d9c2b213029dd126d9f2980506f5a0455c4a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-1-deactivate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True) 2024-09-20T19:50:47.560 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.1.service' 2024-09-20T19:50:47.597 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:50:47.598 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-20T19:50:47.598 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-20T19:50:47.598 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.2 2024-09-20T19:50:47.873 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:47 smithi116 systemd[1]: Stopping Ceph osd.2 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:50:48.373 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:50:47.941+0000 7efffc910640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:50:48.373 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:50:47.941+0000 7efffc910640 -1 osd.2 165 *** Got signal Terminated *** 2024-09-20T19:50:48.373 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:47 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2[73189]: 2024-09-20T19:50:47.941+0000 7efffc910640 -1 osd.2 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:50:53.677 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:53 smithi116 podman[93914]: 2024-09-20 19:50:53.38644772 +0000 UTC m=+5.565421436 container died e0b40360570b5a200ce60ca67443ec3588ee8ae2d2b792f99636812aa1afbfdc (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:50:53.979 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:53 smithi116 podman[93914]: 2024-09-20 19:50:53.749655307 +0000 UTC m=+5.928629020 container cleanup e0b40360570b5a200ce60ca67443ec3588ee8ae2d2b792f99636812aa1afbfdc (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:50:53.979 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:53 smithi116 bash[93914]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2 2024-09-20T19:50:54.373 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:53 smithi116 podman[93947]: 2024-09-20 19:50:53.977844168 +0000 UTC m=+0.594109294 container remove e0b40360570b5a200ce60ca67443ec3588ee8ae2d2b792f99636812aa1afbfdc (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2) 2024-09-20T19:50:54.667 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:54 smithi116 podman[94033]: 2024-09-20 19:50:54.317551947 +0000 UTC m=+0.016704471 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:50:54.668 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:54 smithi116 podman[94033]: 2024-09-20 19:50:54.43549717 +0000 UTC m=+0.134649679 container create 67d8695de305d2a7df9bdd2e96e3bc96983fc62c0dd7fa67d84385362ae28fc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:50:54.668 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:54 smithi116 podman[94033]: 2024-09-20 19:50:54.663067227 +0000 UTC m=+0.362219751 container init 67d8695de305d2a7df9bdd2e96e3bc96983fc62c0dd7fa67d84385362ae28fc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:50:54.668 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:54 smithi116 podman[94033]: 2024-09-20 19:50:54.66638632 +0000 UTC m=+0.365538839 container start 67d8695de305d2a7df9bdd2e96e3bc96983fc62c0dd7fa67d84385362ae28fc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-20T19:50:55.123 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:54 smithi116 podman[94033]: 2024-09-20 19:50:54.77142877 +0000 UTC m=+0.470581289 container attach 67d8695de305d2a7df9bdd2e96e3bc96983fc62c0dd7fa67d84385362ae28fc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:50:55.123 INFO:journalctl@ceph.osd.2.smithi116.stdout:Sep 20 19:50:54 smithi116 podman[94033]: 2024-09-20 19:50:54.860176602 +0000 UTC m=+0.559329122 container died 67d8695de305d2a7df9bdd2e96e3bc96983fc62c0dd7fa67d84385362ae28fc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-20T19:50:55.458 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.2.service' 2024-09-20T19:50:55.501 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:50:55.502 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-20T19:50:55.502 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-20T19:50:55.502 DEBUG:teuthology.orchestra.run.smithi116:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.3 2024-09-20T19:50:55.846 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:50:55 smithi116 systemd[1]: Stopping Ceph osd.3 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:50:56.123 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:50:55 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[75532]: 2024-09-20T19:50:55.844+0000 7fb1300c2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:50:56.123 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:50:55 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[75532]: 2024-09-20T19:50:55.844+0000 7fb1300c2640 -1 osd.3 165 *** Got signal Terminated *** 2024-09-20T19:50:56.123 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:50:55 smithi116 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3[75532]: 2024-09-20T19:50:55.844+0000 7fb1300c2640 -1 osd.3 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:50:59.349 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:50:59 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:50:59.090+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:50:59.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:50:59 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:50:59.348+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:00.464 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:00 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:00.358+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:00.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:00 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:00.117+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:01.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:01 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:01.162+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:01.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:01 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:01.374+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:01.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:01 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:01.218+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:01.623 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:01 smithi116 podman[94133]: 2024-09-20 19:51:01.365545688 +0000 UTC m=+5.628016165 container died e3b204ab779476c968568b7990d08d61d7ca290754c39fc5ef3f6a3381a8c4ac (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True) 2024-09-20T19:51:01.988 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:01 smithi116 podman[94133]: 2024-09-20 19:51:01.744215873 +0000 UTC m=+6.006686349 container cleanup e3b204ab779476c968568b7990d08d61d7ca290754c39fc5ef3f6a3381a8c4ac (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:51:01.988 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:01 smithi116 bash[94133]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3 2024-09-20T19:51:02.131 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:01 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:01.747+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:02.310 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:01 smithi116 podman[94148]: 2024-09-20 19:51:01.987252131 +0000 UTC m=+0.623098596 container remove e3b204ab779476c968568b7990d08d61d7ca290754c39fc5ef3f6a3381a8c4ac (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:51:02.423 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:02 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:02.130+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:02.424 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:02 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:02.267+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:02.623 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:02 smithi116 podman[94239]: 2024-09-20 19:51:02.212273067 +0000 UTC m=+0.016205726 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:51:02.623 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:02 smithi116 podman[94239]: 2024-09-20 19:51:02.321539819 +0000 UTC m=+0.125472462 container create 46d606821ffa4fb100100568fd83a32963a05cac9b7c18fc9b0f5cebd866c438 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-20T19:51:02.623 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:02 smithi116 podman[94239]: 2024-09-20 19:51:02.532291469 +0000 UTC m=+0.336224117 container init 46d606821ffa4fb100100568fd83a32963a05cac9b7c18fc9b0f5cebd866c438 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:51:02.624 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:02 smithi116 podman[94239]: 2024-09-20 19:51:02.535621618 +0000 UTC m=+0.339554264 container start 46d606821ffa4fb100100568fd83a32963a05cac9b7c18fc9b0f5cebd866c438 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, org.label-schema.build-date=20240731) 2024-09-20T19:51:02.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:02 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:02.422+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:03.020 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:02 smithi116 podman[94239]: 2024-09-20 19:51:02.623779822 +0000 UTC m=+0.427712468 container attach 46d606821ffa4fb100100568fd83a32963a05cac9b7c18fc9b0f5cebd866c438 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2) 2024-09-20T19:51:03.020 INFO:journalctl@ceph.osd.3.smithi116.stdout:Sep 20 19:51:02 smithi116 podman[94239]: 2024-09-20 19:51:02.723683292 +0000 UTC m=+0.527615936 container died 46d606821ffa4fb100100568fd83a32963a05cac9b7c18fc9b0f5cebd866c438 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-20T19:51:03.153 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:02 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:02.746+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:03.314 DEBUG:teuthology.orchestra.run.smithi116:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.3.service' 2024-09-20T19:51:03.352 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:51:03.352 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-20T19:51:03.353 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-20T19:51:03.353 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.4 2024-09-20T19:51:03.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:03.278+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:03.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:03.151+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:03.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:03 smithi118 systemd[1]: Stopping Ceph osd.4 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:51:03.466 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:03.375+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:03.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:03.700+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:03.965 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:03.677+0000 7f85689b5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:51:03.965 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:03.677+0000 7f85689b5640 -1 osd.4 165 *** Got signal Terminated *** 2024-09-20T19:51:03.965 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:03 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:03.677+0000 7f85689b5640 -1 osd.4 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:51:04.466 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:04 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:04.250+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:04.466 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:04 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:04.357+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:04.466 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:04 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:04.190+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:05.209 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:04 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:04.715+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:05.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:05 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:05.238+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:05.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:05 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:05.208+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:05.466 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:05 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:05.318+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:05.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:05 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:05.704+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:06.691 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:06 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:06.221+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:06.691 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:06 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:06.216+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:06.692 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:06 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:06.326+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:06.692 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:06 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:06.326+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:06.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:06 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:06.689+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:07.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:07 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:07.222+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:07.715 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:07 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:07.222+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:07.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:07 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:07.370+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:07.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:07 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:07.370+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:08.215 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:07 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:07.730+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:08.714 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:08.233+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:08.714 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:08.356+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:08.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:08.356+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:08.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:08.271+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.856839+0000 front 2024-09-20T19:50:35.857064+0000 (oldest deadline 2024-09-20T19:50:58.756908+0000) 2024-09-20T19:51:08.715 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4[53159]: 2024-09-20T19:51:08.271+0000 7f8564fca640 -1 osd.4 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.657940+0000 front 2024-09-20T19:50:41.657856+0000 (oldest deadline 2024-09-20T19:51:07.458107+0000) 2024-09-20T19:51:08.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:08.712+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:08.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:08 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:08.712+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:09.522 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:09 smithi118 podman[68180]: 2024-09-20 19:51:09.231737897 +0000 UTC m=+5.674424617 container died 4ae34596cfeaa8eca96b49568232123203453026caa52c7516a5074b8c4b416e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-20T19:51:09.522 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:09.215+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:09.522 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:09.215+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:09.523 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:09.374+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:09.523 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:09.374+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:09.939 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:09 smithi118 podman[68180]: 2024-09-20 19:51:09.66996527 +0000 UTC m=+6.112652022 container cleanup 4ae34596cfeaa8eca96b49568232123203453026caa52c7516a5074b8c4b416e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-20T19:51:09.939 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:09 smithi118 bash[68180]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4 2024-09-20T19:51:09.939 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:09.753+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:09.939 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:09 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:09.753+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:10.206 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:09 smithi118 podman[68196]: 2024-09-20 19:51:09.938041178 +0000 UTC m=+0.709057222 container remove 4ae34596cfeaa8eca96b49568232123203453026caa52c7516a5074b8c4b416e (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=) 2024-09-20T19:51:10.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:10 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:10.328+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:10.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:10 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:10.328+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:10.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:10 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:10.204+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:10.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:10 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:10.204+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:10.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:10 smithi118 podman[68285]: 2024-09-20 19:51:10.188208396 +0000 UTC m=+0.015972796 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:51:10.465 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:10 smithi118 podman[68285]: 2024-09-20 19:51:10.305718915 +0000 UTC m=+0.133483354 container create 88c593e4c0bb48d78b709a98c881bec05d644b14a9f3240968584dba7c3d1e10 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-20T19:51:10.758 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:10 smithi118 podman[68285]: 2024-09-20 19:51:10.567193415 +0000 UTC m=+0.394957821 container init 88c593e4c0bb48d78b709a98c881bec05d644b14a9f3240968584dba7c3d1e10 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, ceph=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:51:10.758 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:10 smithi118 podman[68285]: 2024-09-20 19:51:10.571980913 +0000 UTC m=+0.399745311 container start 88c593e4c0bb48d78b709a98c881bec05d644b14a9f3240968584dba7c3d1e10 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:51:10.759 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:10 smithi118 podman[68285]: 2024-09-20 19:51:10.674945535 +0000 UTC m=+0.502709937 container attach 88c593e4c0bb48d78b709a98c881bec05d644b14a9f3240968584dba7c3d1e10 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-20T19:51:11.062 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:10 smithi118 podman[68285]: 2024-09-20 19:51:10.761830997 +0000 UTC m=+0.589595394 container died 88c593e4c0bb48d78b709a98c881bec05d644b14a9f3240968584dba7c3d1e10 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-20T19:51:11.062 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:10 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:10.756+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:11.062 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:10 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:10.756+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:11.312 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:11.175+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:11.312 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:11.175+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:11.312 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:11 smithi118 podman[68308]: 2024-09-20 19:51:11.295249618 +0000 UTC m=+0.527886377 container remove 88c593e4c0bb48d78b709a98c881bec05d644b14a9f3240968584dba7c3d1e10 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-20T19:51:11.313 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:11 smithi118 systemd[1]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-20T19:51:11.313 INFO:journalctl@ceph.osd.4.smithi118.stdout:Sep 20 19:51:11 smithi118 systemd[1]: Stopped Ceph osd.4 for 2d2015e0-7786-11ef-baf5-efdc52797490. 2024-09-20T19:51:11.315 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.4.service' 2024-09-20T19:51:11.354 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:51:11.354 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-20T19:51:11.355 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-20T19:51:11.355 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.5 2024-09-20T19:51:11.683 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:11 smithi118 systemd[1]: Stopping Ceph osd.5 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:51:11.683 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:11.323+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:11.683 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:11.323+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:11.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:11.719+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:11.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:11.719+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:11.965 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:11.681+0000 7fed012d2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:51:11.966 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:11.681+0000 7fed012d2640 -1 osd.5 165 *** Got signal Terminated *** 2024-09-20T19:51:11.966 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:11 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:11.681+0000 7fed012d2640 -1 osd.5 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:51:12.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:12 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:12.134+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:12.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:12 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:12.134+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:12.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:12 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:12.301+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:12.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:12 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:12.301+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:12.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:12 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:12.693+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:12.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:12 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:12.693+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:13.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:13 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:13.142+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:13.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:13 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:13.142+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:13.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:13 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:13.283+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:13.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:13 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:13.283+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:13.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:13 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:13.665+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:13.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:13 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:13.665+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:14.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:14 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:14.162+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:14.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:14 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:14.162+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:14.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:14 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:14.241+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:14.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:14 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:14.241+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:14.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:14 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:14.655+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:14.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:14 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:14.655+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:15.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:15 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:15.150+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:15.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:15 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:15.150+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:15.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:15 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:15.200+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:15.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:15 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:15.200+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:15.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:15 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:15.662+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:15.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:15 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:15.662+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:15.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:15 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:15.662+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:16.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:16.122+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.831700+0000 front 2024-09-20T19:50:33.831663+0000 (oldest deadline 2024-09-20T19:51:00.231908+0000) 2024-09-20T19:51:16.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5[55158]: 2024-09-20T19:51:16.122+0000 7fecf98df640 -1 osd.5 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:43.633192+0000 front 2024-09-20T19:50:43.633181+0000 (oldest deadline 2024-09-20T19:51:08.933059+0000) 2024-09-20T19:51:16.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:16.192+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:16.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:16.192+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:16.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:16.192+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:16.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:16.638+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:16.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:16.638+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:16.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:16 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:16.638+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:17.465 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:17 smithi118 podman[68389]: 2024-09-20 19:51:17.109563813 +0000 UTC m=+5.520571315 container died e6a039aa388ff69fb822691eb29312ebea5fb825e68d173f2536d59da317020a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-20T19:51:17.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:17 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:17.241+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:17.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:17 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:17.241+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:17.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:17 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:17.241+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:17.734 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:17 smithi118 podman[68389]: 2024-09-20 19:51:17.4814322 +0000 UTC m=+5.892439747 container cleanup e6a039aa388ff69fb822691eb29312ebea5fb825e68d173f2536d59da317020a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2) 2024-09-20T19:51:17.734 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:17 smithi118 bash[68389]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5 2024-09-20T19:51:17.734 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:17 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:17.684+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:17.734 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:17 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:17.684+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:17.734 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:17 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:17.684+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:18.089 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:17 smithi118 podman[68431]: 2024-09-20 19:51:17.733070227 +0000 UTC m=+0.621631556 container remove e6a039aa388ff69fb822691eb29312ebea5fb825e68d173f2536d59da317020a (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:18 smithi118 podman[68519]: 2024-09-20 19:51:17.99049462 +0000 UTC m=+0.015738660 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:18 smithi118 podman[68519]: 2024-09-20 19:51:18.100423622 +0000 UTC m=+0.125667649 container create 5936379dca7308ea153e68e363af724ef364ec7f9fe717c7563fb544d14d2d14 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:18 smithi118 podman[68519]: 2024-09-20 19:51:18.319624312 +0000 UTC m=+0.344868344 container init 5936379dca7308ea153e68e363af724ef364ec7f9fe717c7563fb544d14d2d14 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True) 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:18 smithi118 podman[68519]: 2024-09-20 19:51:18.322880989 +0000 UTC m=+0.348125027 container start 5936379dca7308ea153e68e363af724ef364ec7f9fe717c7563fb544d14d2d14 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:18 smithi118 podman[68519]: 2024-09-20 19:51:18.377623797 +0000 UTC m=+0.402867839 container attach 5936379dca7308ea153e68e363af724ef364ec7f9fe717c7563fb544d14d2d14 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:18 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:18.213+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:18 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:18.213+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:18.379 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:18 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:18.213+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:18.670 INFO:journalctl@ceph.osd.5.smithi118.stdout:Sep 20 19:51:18 smithi118 podman[68519]: 2024-09-20 19:51:18.507040446 +0000 UTC m=+0.532284484 container died 5936379dca7308ea153e68e363af724ef364ec7f9fe717c7563fb544d14d2d14 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T19:51:18.671 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:18 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:18.668+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:18.671 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:18 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:18.668+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:18.671 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:18 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:18.668+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:19.102 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.5.service' 2024-09-20T19:51:19.142 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:51:19.142 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-20T19:51:19.142 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-20T19:51:19.142 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.6 2024-09-20T19:51:19.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:19.199+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:19.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:19.199+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:19.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:19.199+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:19.465 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:19 smithi118 systemd[1]: Stopping Ceph osd.6 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:51:19.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:19.676+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:19.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:19.676+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:19.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:19.676+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:19.965 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:19.469+0000 7fc56d33d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:51:19.965 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:19.469+0000 7fc56d33d640 -1 osd.6 165 *** Got signal Terminated *** 2024-09-20T19:51:19.965 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:19 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:19.469+0000 7fc56d33d640 -1 osd.6 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:51:20.676 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:20.228+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:20.676 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:20.228+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:20.676 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:20.228+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:20.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:20.674+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:20.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:20.674+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:20.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:20 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:20.674+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:21.644 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:21 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:21.255+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:21.644 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:21 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:21.255+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:21.644 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:21 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:21.255+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:21.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:21 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:21.641+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:21.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:21 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:21.641+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:21.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:21 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:21.641+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:22.562 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:22.274+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:22.562 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:22.274+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:22.562 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:22.274+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:22.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:22.686+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:22.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:22.686+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:22.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:22 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:22.686+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:23.700 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:23.299+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:23.700 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:23.299+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:23.700 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:23.299+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:23.700 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:23.299+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.641369+0000 front 2024-09-20T19:51:00.641394+0000 (oldest deadline 2024-09-20T19:51:22.941366+0000) 2024-09-20T19:51:23.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:23.697+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:23.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:23.697+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:23.966 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:23.697+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:23.966 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:23 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:23.697+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:24.677 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:24.264+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:33.337867+0000 front 2024-09-20T19:50:33.337798+0000 (oldest deadline 2024-09-20T19:50:58.637629+0000) 2024-09-20T19:51:24.677 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:24.264+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:44.439187+0000 front 2024-09-20T19:50:44.439174+0000 (oldest deadline 2024-09-20T19:51:06.138967+0000) 2024-09-20T19:51:24.678 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:24.264+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.440818+0000 front 2024-09-20T19:50:52.440746+0000 (oldest deadline 2024-09-20T19:51:15.340665+0000) 2024-09-20T19:51:24.678 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6[57166]: 2024-09-20T19:51:24.264+0000 7fc56594a640 -1 osd.6 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.641369+0000 front 2024-09-20T19:51:00.641394+0000 (oldest deadline 2024-09-20T19:51:22.941366+0000) 2024-09-20T19:51:24.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:24.675+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:24.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:24.675+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:24.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:24.675+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:24.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:24 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:24.675+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:25.292 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:25 smithi118 podman[68621]: 2024-09-20 19:51:25.009885488 +0000 UTC m=+5.643986337 container died 14b930edda7f29b20eb0c28cbc0996b3dcb8054b79916a126a345561c019f214 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-20T19:51:25.657 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:25 smithi118 podman[68621]: 2024-09-20 19:51:25.465851568 +0000 UTC m=+6.099952416 container cleanup 14b930edda7f29b20eb0c28cbc0996b3dcb8054b79916a126a345561c019f214 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-20T19:51:25.657 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:25 smithi118 bash[68621]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6 2024-09-20T19:51:25.965 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:25 smithi118 podman[68637]: 2024-09-20 19:51:25.7431577 +0000 UTC m=+0.732449818 container remove 14b930edda7f29b20eb0c28cbc0996b3dcb8054b79916a126a345561c019f214 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:51:25.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:25 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:25.655+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:25.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:25 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:25.655+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:25.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:25 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:25.655+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:25.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:25 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:25.655+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:26.389 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:26 smithi118 podman[68727]: 2024-09-20 19:51:25.983505223 +0000 UTC m=+0.015830776 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:51:26.389 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:26 smithi118 podman[68727]: 2024-09-20 19:51:26.085476585 +0000 UTC m=+0.117802146 container create ae1449be87c0871789dd18151f040b452b47e96626b441d38fb1403de7dd2d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-20T19:51:26.389 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:26 smithi118 podman[68727]: 2024-09-20 19:51:26.287794014 +0000 UTC m=+0.320119569 container init ae1449be87c0871789dd18151f040b452b47e96626b441d38fb1403de7dd2d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-20T19:51:26.389 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:26 smithi118 podman[68727]: 2024-09-20 19:51:26.291641172 +0000 UTC m=+0.323966726 container start ae1449be87c0871789dd18151f040b452b47e96626b441d38fb1403de7dd2d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T19:51:26.389 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:26 smithi118 podman[68727]: 2024-09-20 19:51:26.387738139 +0000 UTC m=+0.420063712 container attach ae1449be87c0871789dd18151f040b452b47e96626b441d38fb1403de7dd2d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T19:51:26.715 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:26 smithi118 podman[68727]: 2024-09-20 19:51:26.481804853 +0000 UTC m=+0.514130423 container died ae1449be87c0871789dd18151f040b452b47e96626b441d38fb1403de7dd2d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-20T19:51:26.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:26 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:26.611+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:26.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:26 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:26.611+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:26.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:26 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:26.611+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:26.716 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:26 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:26.611+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:27.036 INFO:journalctl@ceph.osd.6.smithi118.stdout:Sep 20 19:51:27 smithi118 podman[68750]: 2024-09-20 19:51:27.033112073 +0000 UTC m=+0.545507610 container remove ae1449be87c0871789dd18151f040b452b47e96626b441d38fb1403de7dd2d8c (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-20T19:51:27.054 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.6.service' 2024-09-20T19:51:27.093 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:51:27.093 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-20T19:51:27.093 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-20T19:51:27.093 DEBUG:teuthology.orchestra.run.smithi118:> sudo systemctl stop ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.7 2024-09-20T19:51:27.413 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 systemd[1]: Stopping Ceph osd.7 for 2d2015e0-7786-11ef-baf5-efdc52797490... 2024-09-20T19:51:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:27.410+0000 7f99afc54640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T19:51:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:27.410+0000 7f99afc54640 -1 osd.7 165 *** Got signal Terminated *** 2024-09-20T19:51:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:27.410+0000 7f99afc54640 -1 osd.7 165 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T19:51:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:27.604+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:27.604+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:27.604+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:27.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:27 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:27.604+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:28.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:28 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:28.571+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:28.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:28 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:28.571+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:28.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:28 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:28.571+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:28.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:28 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:28.571+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:29.866 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:29 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:29.561+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:29.866 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:29 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:29.561+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:29.866 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:29 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:29.561+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:29.866 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:29 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:29.561+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:30.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:30.599+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:30.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:30.599+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:30.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:30.599+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:30.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:30.599+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:30.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:30 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:30.599+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.118:6804 osd.4 since back 2024-09-20T19:51:06.261694+0000 front 2024-09-20T19:51:06.261770+0000 (oldest deadline 2024-09-20T19:51:30.361626+0000) 2024-09-20T19:51:31.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:31 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:31.580+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6806 osd.0 since back 2024-09-20T19:50:35.358123+0000 front 2024-09-20T19:50:35.358234+0000 (oldest deadline 2024-09-20T19:51:01.158465+0000) 2024-09-20T19:51:31.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:31 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:31.580+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6814 osd.1 since back 2024-09-20T19:50:41.159059+0000 front 2024-09-20T19:50:41.158862+0000 (oldest deadline 2024-09-20T19:51:08.159333+0000) 2024-09-20T19:51:31.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:31 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:31.580+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6822 osd.2 since back 2024-09-20T19:50:52.260225+0000 front 2024-09-20T19:50:52.260208+0000 (oldest deadline 2024-09-20T19:51:15.160103+0000) 2024-09-20T19:51:31.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:31 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:31.580+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.116:6830 osd.3 since back 2024-09-20T19:51:00.460956+0000 front 2024-09-20T19:51:00.461016+0000 (oldest deadline 2024-09-20T19:51:22.760790+0000) 2024-09-20T19:51:31.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:31 smithi118 ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7[59169]: 2024-09-20T19:51:31.580+0000 7f99ac269640 -1 osd.7 165 heartbeat_check: no reply from 172.21.15.118:6804 osd.4 since back 2024-09-20T19:51:06.261694+0000 front 2024-09-20T19:51:06.261770+0000 (oldest deadline 2024-09-20T19:51:30.361626+0000) 2024-09-20T19:51:33.260 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:32 smithi118 podman[68828]: 2024-09-20 19:51:32.997038652 +0000 UTC m=+5.683891290 container died d60081632e22113859dd3e0a3ecdec336dc904036c51171580053c7e52264e54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:51:33.631 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:33 smithi118 podman[68828]: 2024-09-20 19:51:33.426766969 +0000 UTC m=+6.113619600 container cleanup d60081632e22113859dd3e0a3ecdec336dc904036c51171580053c7e52264e54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-20T19:51:33.631 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:33 smithi118 bash[68828]: ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7 2024-09-20T19:51:33.965 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:33 smithi118 podman[68855]: 2024-09-20 19:51:33.629556319 +0000 UTC m=+0.632160431 container remove d60081632e22113859dd3e0a3ecdec336dc904036c51171580053c7e52264e54 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-20T19:51:34.370 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:34 smithi118 podman[68962]: 2024-09-20 19:51:33.945066733 +0000 UTC m=+0.015798263 image pull 82f5f45939a4d8f62db4f02ae9374981c40785acf9bb0102013462d795122b30 quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d 2024-09-20T19:51:34.370 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:34 smithi118 podman[68962]: 2024-09-20 19:51:34.082123801 +0000 UTC m=+0.152855397 container create 7005a33e7a3e00499bdae61a7644a0d23a40fadd6bb77131dc8964e5c30e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1) 2024-09-20T19:51:34.370 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:34 smithi118 podman[68962]: 2024-09-20 19:51:34.286271878 +0000 UTC m=+0.357003414 container init 7005a33e7a3e00499bdae61a7644a0d23a40fadd6bb77131dc8964e5c30e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-20T19:51:34.370 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:34 smithi118 podman[68962]: 2024-09-20 19:51:34.28972797 +0000 UTC m=+0.360459491 container start 7005a33e7a3e00499bdae61a7644a0d23a40fadd6bb77131dc8964e5c30e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-20T19:51:34.370 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:34 smithi118 podman[68962]: 2024-09-20 19:51:34.368940769 +0000 UTC m=+0.439672290 container attach 7005a33e7a3e00499bdae61a7644a0d23a40fadd6bb77131dc8964e5c30e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-20T19:51:34.715 INFO:journalctl@ceph.osd.7.smithi118.stdout:Sep 20 19:51:34 smithi118 podman[68962]: 2024-09-20 19:51:34.48482245 +0000 UTC m=+0.555553973 container died 7005a33e7a3e00499bdae61a7644a0d23a40fadd6bb77131dc8964e5c30e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:dc9dc647e3e3360d1adc5cf221a5bec436cbf643c6188f083c790ea05138208d, name=ceph-2d2015e0-7786-11ef-baf5-efdc52797490-osd-7-deactivate, RELEASE=wip-adk-testing-2024-09-20-0910-e8baca1, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-20T19:51:35.182 DEBUG:teuthology.orchestra.run.smithi118:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2d2015e0-7786-11ef-baf5-efdc52797490@osd.7.service' 2024-09-20T19:51:35.219 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T19:51:35.220 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-20T19:51:35.220 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-20T19:54:33.820 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-20T19:54:52.804 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T19:54:52.839 DEBUG:teuthology.orchestra.run.smithi118:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T19:54:52.866 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-20T19:54:52.868 DEBUG:teuthology.misc:Transferring archived files from smithi116:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786/remote/smithi116/crash 2024-09-20T19:54:52.869 DEBUG:teuthology.orchestra.run.smithi116:> sudo tar c -f - -C /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/crash -- . 2024-09-20T19:54:52.916 INFO:teuthology.orchestra.run.smithi116.stderr:tar: /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-20T19:54:52.916 INFO:teuthology.orchestra.run.smithi116.stderr:tar: Error is not recoverable: exiting now 2024-09-20T19:54:52.918 DEBUG:teuthology.misc:Transferring archived files from smithi118:/var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786/remote/smithi118/crash 2024-09-20T19:54:52.919 DEBUG:teuthology.orchestra.run.smithi118:> sudo tar c -f - -C /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/crash -- . 2024-09-20T19:54:52.953 INFO:teuthology.orchestra.run.smithi118.stderr:tar: /var/lib/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-20T19:54:52.953 INFO:teuthology.orchestra.run.smithi118.stderr:tar: Error is not recoverable: exiting now 2024-09-20T19:54:52.954 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-20T19:54:52.955 DEBUG:teuthology.orchestra.run.smithi116:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-20T19:54:52.987 INFO:tasks.cephadm:Compressing logs... 2024-09-20T19:54:52.987 DEBUG:teuthology.orchestra.run.smithi116:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-20T19:54:53.030 DEBUG:teuthology.orchestra.run.smithi118:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-20T19:54:53.054 INFO:teuthology.orchestra.run.smithi116.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.0.log 2024-09-20T19:54:53.054 INFO:teuthology.orchestra.run.smithi116.stderr:: No such file or directory 2024-09-20T19:54:53.054 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.1.log 2024-09-20T19:54:53.054 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-20T19:54:53.054 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.0.log: /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mon.smithi116.log 2024-09-20T19:54:53.055 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mgr.smithi118.rmiazv.log 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:: No such file or directory 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.7.log 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mgr.smithi118.rmiazv.log: /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.4.log 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-20T19:54:53.058 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.4.log: /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.6.log 2024-09-20T19:54:53.059 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-20T19:54:53.059 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mon.smithi116.log: /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.2.log 2024-09-20T19:54:53.059 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.log 2024-09-20T19:54:53.059 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mgr.smithi116.poxcrq.log 2024-09-20T19:54:53.059 INFO:teuthology.orchestra.run.smithi116.stderr: 85.3% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.cephadm.log.gz/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.log: 2024-09-20T19:54:53.060 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.5.log 2024-09-20T19:54:53.060 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.6.log: /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.5.log: 85.3% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-20T19:54:53.060 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.log 2024-09-20T19:54:53.060 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mon.smithi118.log 2024-09-20T19:54:53.060 INFO:teuthology.orchestra.run.smithi118.stderr: 91.7% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-20T19:54:53.061 INFO:teuthology.orchestra.run.smithi116.stderr: 91.5% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-20T19:54:53.061 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi116.xvfgaf.log 2024-09-20T19:54:53.061 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mgr.smithi116.poxcrq.log: 87.0%gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.3.log 2024-09-20T19:54:53.061 INFO:teuthology.orchestra.run.smithi116.stderr: -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-20T19:54:53.062 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi116.xvfgaf.log: 89.4% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi116.xvfgaf.log.gz 2024-09-20T19:54:53.062 INFO:teuthology.orchestra.run.smithi116.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-20T19:54:53.062 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi118.zumcyi.log 2024-09-20T19:54:53.064 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mon.smithi118.log: 87.0% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-20T19:54:53.067 INFO:teuthology.orchestra.run.smithi116.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.1% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-20T19:54:53.069 INFO:teuthology.orchestra.run.smithi118.stderr: 91.2% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-20T19:54:53.069 INFO:teuthology.orchestra.run.smithi118.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-20T19:54:53.070 INFO:teuthology.orchestra.run.smithi118.stderr:/var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi118.zumcyi.log: /var/log/ceph/cephadm.log: 88.0% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi118.zumcyi.log.gz 2024-09-20T19:54:53.077 INFO:teuthology.orchestra.run.smithi118.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-20T19:54:53.090 INFO:teuthology.orchestra.run.smithi116.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-20T19:54:53.117 INFO:teuthology.orchestra.run.smithi118.stderr: 89.8% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mgr.smithi118.rmiazv.log.gz 2024-09-20T19:54:53.349 INFO:teuthology.orchestra.run.smithi116.stderr: 88.6% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mgr.smithi116.poxcrq.log.gz 2024-09-20T19:54:53.536 INFO:teuthology.orchestra.run.smithi118.stderr: 92.6% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mon.smithi118.log.gz 2024-09-20T19:54:54.453 INFO:teuthology.orchestra.run.smithi116.stderr: 91.8% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-mon.smithi116.log.gz 2024-09-20T19:55:04.595 INFO:teuthology.orchestra.run.smithi116.stderr: 94.5% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.2.log.gz 2024-09-20T19:55:06.071 INFO:teuthology.orchestra.run.smithi118.stderr: 94.0% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.4.log.gz 2024-09-20T19:55:08.650 INFO:teuthology.orchestra.run.smithi118.stderr: 94.2% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.6.log.gz 2024-09-20T19:55:09.098 INFO:teuthology.orchestra.run.smithi116.stderr: 94.1% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.0.log.gz 2024-09-20T19:55:11.211 INFO:teuthology.orchestra.run.smithi116.stderr: 94.1% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.1.log.gz 2024-09-20T19:55:12.549 INFO:teuthology.orchestra.run.smithi118.stderr: 94.2% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.5.log.gz 2024-09-20T19:55:12.575 INFO:teuthology.orchestra.run.smithi116.stderr: 94.0% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.3.log.gz 2024-09-20T19:55:12.577 INFO:teuthology.orchestra.run.smithi116.stderr: 2024-09-20T19:55:12.577 INFO:teuthology.orchestra.run.smithi116.stderr:real 0m19.534s 2024-09-20T19:55:12.577 INFO:teuthology.orchestra.run.smithi116.stderr:user 1m4.316s 2024-09-20T19:55:12.577 INFO:teuthology.orchestra.run.smithi116.stderr:sys 0m2.570s 2024-09-20T19:55:15.222 INFO:teuthology.orchestra.run.smithi118.stderr: 94.1% -- replaced with /var/log/ceph/2d2015e0-7786-11ef-baf5-efdc52797490/ceph-osd.7.log.gz 2024-09-20T19:55:15.224 INFO:teuthology.orchestra.run.smithi118.stderr: 2024-09-20T19:55:15.224 INFO:teuthology.orchestra.run.smithi118.stderr:real 0m22.179s 2024-09-20T19:55:15.224 INFO:teuthology.orchestra.run.smithi118.stderr:user 1m8.043s 2024-09-20T19:55:15.224 INFO:teuthology.orchestra.run.smithi118.stderr:sys 0m2.676s 2024-09-20T19:55:15.224 INFO:tasks.cephadm:Archiving logs... 2024-09-20T19:55:15.225 DEBUG:teuthology.misc:Transferring archived files from smithi116:/var/log/ceph to /home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786/remote/smithi116/log 2024-09-20T19:55:15.226 DEBUG:teuthology.orchestra.run.smithi116:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-20T19:55:23.782 DEBUG:teuthology.misc:Transferring archived files from smithi118:/var/log/ceph to /home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786/remote/smithi118/log 2024-09-20T19:55:23.783 DEBUG:teuthology.orchestra.run.smithi118:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-20T19:55:33.245 INFO:tasks.cephadm:Removing cluster... 2024-09-20T19:55:33.245 DEBUG:teuthology.orchestra.run.smithi116:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 --force 2024-09-20T19:55:33.606 DEBUG:teuthology.orchestra.run.smithi118:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2d2015e0-7786-11ef-baf5-efdc52797490 --force 2024-09-20T19:55:33.978 INFO:tasks.cephadm:Removing cephadm ... 2024-09-20T19:55:33.979 DEBUG:teuthology.orchestra.run.smithi116:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-20T19:55:33.995 DEBUG:teuthology.orchestra.run.smithi118:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-20T19:55:34.010 INFO:tasks.cephadm:Teardown complete 2024-09-20T19:55:34.010 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-20T19:55:34.020 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-20T19:55:34.020 DEBUG:teuthology.orchestra.run.smithi116:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T19:55:34.039 DEBUG:teuthology.orchestra.run.smithi118:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T19:55:34.051 INFO:teuthology.orchestra.run.smithi116.stderr:bash: line 1: ntpq: command not found 2024-09-20T19:55:34.055 INFO:teuthology.orchestra.run.smithi116.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T19:55:34.055 INFO:teuthology.orchestra.run.smithi116.stdout:=============================================================================== 2024-09-20T19:55:34.055 INFO:teuthology.orchestra.run.smithi116.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 16 +594us[ +594us] +/- 43ms 2024-09-20T19:55:34.055 INFO:teuthology.orchestra.run.smithi116.stdout:^* hv02.front.sepia.ceph.com 3 6 377 18 +512us[ +564us] +/- 58ms 2024-09-20T19:55:34.055 INFO:teuthology.orchestra.run.smithi116.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 18 -1713us[-1713us] +/- 79ms 2024-09-20T19:55:34.055 INFO:teuthology.orchestra.run.smithi116.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:55:34.064 INFO:teuthology.orchestra.run.smithi118.stderr:bash: line 1: ntpq: command not found 2024-09-20T19:55:34.069 INFO:teuthology.orchestra.run.smithi118.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T19:55:34.069 INFO:teuthology.orchestra.run.smithi118.stdout:=============================================================================== 2024-09-20T19:55:34.069 INFO:teuthology.orchestra.run.smithi118.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 21 +588us[ +623us] +/- 43ms 2024-09-20T19:55:34.070 INFO:teuthology.orchestra.run.smithi118.stdout:^* hv02.front.sepia.ceph.com 3 6 377 20 +518us[ +553us] +/- 58ms 2024-09-20T19:55:34.070 INFO:teuthology.orchestra.run.smithi118.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 19 -1717us[-1717us] +/- 79ms 2024-09-20T19:55:34.070 INFO:teuthology.orchestra.run.smithi118.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T19:55:34.070 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-20T19:55:34.079 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-20T19:55:34.079 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-20T19:55:34.088 DEBUG:teuthology.orchestra.run.smithi116:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-20T19:55:34.171 DEBUG:teuthology.orchestra.run.smithi118:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-20T19:55:34.241 DEBUG:teuthology.orchestra.run.smithi116:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T19:55:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:55:34.270 DEBUG:teuthology.orchestra.run.smithi118:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T19:55:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:55:34.297 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-20T19:55:34.307 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-20T19:55:34.316 INFO:teuthology.task.internal:Duration was 1992.582234 seconds 2024-09-20T19:55:34.316 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-20T19:55:34.325 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-20T19:55:34.325 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-20T19:55:34.328 DEBUG:teuthology.orchestra.run.smithi118:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-20T19:55:34.364 INFO:teuthology.orchestra.run.smithi116.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T19:55:34.375 INFO:teuthology.orchestra.run.smithi118.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T19:55:34.749 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-20T19:55:34.750 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi116.front.sepia.ceph.com 2024-09-20T19:55:34.750 DEBUG:teuthology.orchestra.run.smithi116:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-20T19:55:34.770 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi118.front.sepia.ceph.com 2024-09-20T19:55:34.770 DEBUG:teuthology.orchestra.run.smithi118:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-20T19:55:34.812 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-20T19:55:34.812 DEBUG:teuthology.orchestra.run.smithi116:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-20T19:55:34.815 DEBUG:teuthology.orchestra.run.smithi118:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-20T19:55:34.878 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-20T19:55:34.878 DEBUG:teuthology.orchestra.run.smithi116:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-20T19:55:34.882 DEBUG:teuthology.orchestra.run.smithi118:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-20T19:55:48.626 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-20T19:55:48.640 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-20T19:55:48.640 DEBUG:teuthology.orchestra.run.smithi116:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-20T19:55:48.665 DEBUG:teuthology.orchestra.run.smithi118:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-20T19:55:48.696 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-20T19:55:48.706 DEBUG:teuthology.orchestra.run.smithi116:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-20T19:55:48.708 DEBUG:teuthology.orchestra.run.smithi118:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-20T19:55:48.729 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern = core 2024-09-20T19:55:48.765 INFO:teuthology.orchestra.run.smithi118.stdout:kernel.core_pattern = core 2024-09-20T19:55:48.791 DEBUG:teuthology.orchestra.run.smithi116:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-20T19:55:48.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:55:48.805 DEBUG:teuthology.orchestra.run.smithi118:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-20T19:55:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T19:55:48.849 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-20T19:55:48.868 INFO:teuthology.task.internal:Transferring archived files... 2024-09-20T19:55:48.869 DEBUG:teuthology.misc:Transferring archived files from smithi116:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786/remote/smithi116 2024-09-20T19:55:48.869 DEBUG:teuthology.orchestra.run.smithi116:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-20T19:55:48.937 DEBUG:teuthology.misc:Transferring archived files from smithi118:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-09-20_18:25:04-orch:cephadm-wip-adk-testing-2024-09-20-0910-distro-default-smithi/7912786/remote/smithi118 2024-09-20T19:55:48.937 DEBUG:teuthology.orchestra.run.smithi118:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-20T19:55:48.985 INFO:teuthology.task.internal:Removing archive directory... 2024-09-20T19:55:48.985 DEBUG:teuthology.orchestra.run.smithi116:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-20T19:55:48.987 DEBUG:teuthology.orchestra.run.smithi118:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-20T19:55:49.036 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-20T19:55:49.050 INFO:teuthology.task.internal:Not uploading archives. 2024-09-20T19:55:49.050 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-20T19:55:49.068 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-20T19:55:49.069 DEBUG:teuthology.orchestra.run.smithi116:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-20T19:55:49.071 DEBUG:teuthology.orchestra.run.smithi118:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-20T19:55:49.084 INFO:teuthology.orchestra.run.smithi116.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 20 19:55 /home/ubuntu/cephtest 2024-09-20T19:55:49.094 INFO:teuthology.orchestra.run.smithi118.stdout: 266063 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 20 19:55 /home/ubuntu/cephtest 2024-09-20T19:55:49.095 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-20T19:55:49.106 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-20T19:55:49.129 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1992.5822339057922 owner: scheduled_adking@teuthology success: true 2024-09-20T19:55:49.130 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-20T19:55:49.213 INFO:teuthology.run:pass