2024-04-25T04:05:20.361 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-25T04:05:20.362 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_adking@teuthology --verbose --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --archive /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738 --name adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi -- /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738/orig.config.yaml 2024-04-25T04:05:20.397 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-25T04:05:20.476 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738 branch: wip-adk-testing-2024-04-24-1625 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7672738' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-04-24-1625 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-adk-testing-2024-04-24-1625 sha1: d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 8141 sha1: d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 sleep_before_teardown: 0 subset: 1/16 suite: orch:cephadm suite_branch: wip-adk-testing-2024-04-24-1625 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDHhiTByJTBufRoF3HnPiFPO3P1m+eRoBBAaeRMOBtAmb+lP/xsB/OC7/VevKbhvNOj69+5FCmu5Z1chLpDoxt8= smithi112.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKDsO+Zoh4YK1/lDiyVjr+1tw1BQH+afbeqZqn9RkBuX8FZKvpBgaE4OyA22Oq88BEJCl0LIV2E9BjABncAFp48= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-25_03:53:55 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-25T04:05:20.476 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737/qa; will attempt to use it 2024-04-25T04:05:20.477 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737/qa/tasks 2024-04-25T04:05:20.477 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-25T04:05:20.478 INFO:teuthology.task.internal:Checking packages... 2024-04-25T04:05:20.501 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd6f5f4e5ec08db21b2896cb71b4cb907b7fb4737' 2024-04-25T04:05:20.501 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-25T04:05:20.501 INFO:teuthology.packaging:ref: None 2024-04-25T04:05:20.501 INFO:teuthology.packaging:tag: None 2024-04-25T04:05:20.501 INFO:teuthology.packaging:branch: wip-adk-testing-2024-04-24-1625 2024-04-25T04:05:20.501 INFO:teuthology.packaging:sha1: d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 2024-04-25T04:05:20.501 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-04-24-1625 2024-04-25T04:05:20.646 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3293.gd6f5f4e5 2024-04-25T04:05:20.648 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-25T04:05:20.653 INFO:teuthology.task.internal:no buildpackages task found 2024-04-25T04:05:20.653 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-25T04:05:20.667 INFO:teuthology.task.internal:Saving configuration 2024-04-25T04:05:20.680 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-25T04:05:20.685 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-25T04:05:20.710 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 04:00:28.261406', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDHhiTByJTBufRoF3HnPiFPO3P1m+eRoBBAaeRMOBtAmb+lP/xsB/OC7/VevKbhvNOj69+5FCmu5Z1chLpDoxt8='} 2024-04-25T04:05:20.730 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi112.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-25 04:00:28.259950', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKDsO+Zoh4YK1/lDiyVjr+1tw1BQH+afbeqZqn9RkBuX8FZKvpBgaE4OyA22Oq88BEJCl0LIV2E9BjABncAFp48='} 2024-04-25T04:05:20.730 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-25T04:05:20.738 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-25T04:05:20.738 INFO:teuthology.task.internal:roles: ubuntu@smithi112.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-25T04:05:20.738 INFO:teuthology.run_tasks:Running task console_log... 2024-04-25T04:05:20.805 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f5ca16c0160>, signals=[15]) 2024-04-25T04:05:20.806 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-25T04:05:20.811 INFO:teuthology.task.internal:Opening connections... 2024-04-25T04:05:20.812 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:05:20.813 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T04:05:20.886 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi112.front.sepia.ceph.com 2024-04-25T04:05:20.887 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi112.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T04:05:20.962 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-25T04:05:20.968 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-04-25T04:05:20.986 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-04-25T04:05:20.986 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-04-25T04:05:21.042 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-04-25T04:05:21.042 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-04-25T04:05:21.042 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-04-25T04:05:21.042 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-04-25T04:05:21.042 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-25T04:05:21.043 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-25T04:05:21.044 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-04-25T04:05:21.062 DEBUG:teuthology.orchestra.run.smithi112:> uname -m 2024-04-25T04:05:21.080 INFO:teuthology.orchestra.run.smithi112.stdout:x86_64 2024-04-25T04:05:21.080 DEBUG:teuthology.orchestra.run.smithi112:> cat /etc/os-release 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:NAME="CentOS Stream" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:VERSION="9" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:ID="centos" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:ID_LIKE="rhel fedora" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:VERSION_ID="9" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:PLATFORM_ID="platform:el9" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:ANSI_COLOR="0;31" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:LOGO="fedora-logo-icon" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:HOME_URL="https://centos.org/" 2024-04-25T04:05:21.138 INFO:teuthology.orchestra.run.smithi112.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-25T04:05:21.139 INFO:teuthology.orchestra.run.smithi112.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-25T04:05:21.139 INFO:teuthology.orchestra.run.smithi112.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-25T04:05:21.139 INFO:teuthology.lock.ops:Updating smithi112.front.sepia.ceph.com on lock server 2024-04-25T04:05:21.161 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-25T04:05:21.170 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-25T04:05:21.175 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-25T04:05:21.175 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-04-25T04:05:21.177 DEBUG:teuthology.orchestra.run.smithi112:> test '!' -e /home/ubuntu/cephtest 2024-04-25T04:05:21.194 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-25T04:05:21.200 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-25T04:05:21.200 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-04-25T04:05:21.234 DEBUG:teuthology.orchestra.run.smithi112:> test -z $(ls -A /var/lib/ceph) 2024-04-25T04:05:21.266 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-25T04:05:21.317 INFO:teuthology.run_tasks:Running task kernel... 2024-04-25T04:05:21.331 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-25T04:05:21.331 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-25T04:05:21.332 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-25T04:05:21.332 DEBUG:teuthology.orchestra.run.smithi112:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-25T04:05:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:05:21.348 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-04-25T04:05:21.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:05:21.349 DEBUG:teuthology.orchestra.run.smithi112:> uname -r 2024-04-25T04:05:21.404 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-437.el9.x86_64 2024-04-25T04:05:21.404 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-437.el9.x86_64 2024-04-25T04:05:21.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-04-25T04:05:21.406 INFO:teuthology.orchestra.run.smithi112.stdout:5.14.0-437.el9.x86_64 2024-04-25T04:05:21.407 INFO:teuthology.task.kernel:Running kernel on smithi112: 5.14.0-437.el9.x86_64 2024-04-25T04:05:21.407 DEBUG:teuthology.orchestra.run.smithi112:> sudo yum install -y kernel 2024-04-25T04:05:23.773 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.0 MB 00:00 2024-04-25T04:05:26.277 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - BaseOS 2.4 MB/s | 8.0 MB 00:03 2024-04-25T04:05:28.477 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - AppStream 17 MB/s | 19 MB 00:01 2024-04-25T04:05:32.520 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.2 MB 00:00 2024-04-25T04:05:34.246 INFO:teuthology.orchestra.run.smithi112.stdout:CentOS Stream 9 - Extras packages 41 kB/s | 16 kB 00:00 2024-04-25T04:05:37.389 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 1.5 MB/s | 19 MB 00:12 2024-04-25T04:05:41.438 INFO:teuthology.orchestra.run.smithi112.stdout:Extra Packages for Enterprise Linux 3.0 MB/s | 21 MB 00:07 2024-04-25T04:05:41.948 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 7.0 MB/s | 6.2 MB 00:00 2024-04-25T04:05:44.607 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 12 kB/s | 16 kB 00:01 2024-04-25T04:05:46.451 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-04-25T04:05:46.588 INFO:teuthology.orchestra.run.smithi112.stdout:lab-extras 16 kB/s | 1.7 kB 00:00 2024-04-25T04:05:48.511 INFO:teuthology.orchestra.run.smithi112.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-25T04:05:48.511 INFO:teuthology.orchestra.run.smithi112.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-25T04:05:48.589 INFO:teuthology.orchestra.run.smithi112.stdout:Dependencies resolved. 2024-04-25T04:05:48.590 INFO:teuthology.orchestra.run.smithi112.stdout:Nothing to do. 2024-04-25T04:05:48.590 INFO:teuthology.orchestra.run.smithi112.stdout:Complete! 2024-04-25T04:05:48.701 DEBUG:teuthology.orchestra.run.smithi112:> echo no | sudo yum reinstall kernel || true 2024-04-25T04:05:49.199 INFO:teuthology.orchestra.run.smithi112.stdout:Last metadata expiration check: 0:00:03 ago on Thu 25 Apr 2024 04:05:46 AM UTC. 2024-04-25T04:05:49.318 INFO:teuthology.orchestra.run.smithi112.stdout:Dependencies resolved. 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout: Package Architecture Version Repository Size 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout:Reinstalling: 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction Summary 2024-04-25T04:05:49.333 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-25T04:05:49.334 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:05:49.334 INFO:teuthology.orchestra.run.smithi112.stdout:Total download size: 255 k 2024-04-25T04:05:49.334 INFO:teuthology.orchestra.run.smithi112.stdout:Installed size: 0 2024-04-25T04:05:49.334 INFO:teuthology.orchestra.run.smithi112.stderr:Operation aborted. 2024-04-25T04:05:49.388 INFO:teuthology.orchestra.run.smithi112.stdout:Is this ok [y/N]: 2024-04-25T04:05:49.389 DEBUG:teuthology.orchestra.run.smithi112:> sudo yum reinstall -y kernel || true 2024-04-25T04:05:49.876 INFO:teuthology.orchestra.run.smithi112.stdout:Last metadata expiration check: 0:00:03 ago on Thu 25 Apr 2024 04:05:46 AM UTC. 2024-04-25T04:05:49.992 INFO:teuthology.orchestra.run.smithi112.stdout:Dependencies resolved. 2024-04-25T04:05:49.993 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-25T04:05:49.994 INFO:teuthology.orchestra.run.smithi112.stdout: Package Architecture Version Repository Size 2024-04-25T04:05:49.994 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-25T04:05:49.994 INFO:teuthology.orchestra.run.smithi112.stdout:Reinstalling: 2024-04-25T04:05:49.994 INFO:teuthology.orchestra.run.smithi112.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-25T04:05:49.994 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:05:49.995 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction Summary 2024-04-25T04:05:49.995 INFO:teuthology.orchestra.run.smithi112.stdout:================================================================================ 2024-04-25T04:05:49.995 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:05:49.995 INFO:teuthology.orchestra.run.smithi112.stdout:Total download size: 255 k 2024-04-25T04:05:49.995 INFO:teuthology.orchestra.run.smithi112.stdout:Installed size: 0 2024-04-25T04:05:49.995 INFO:teuthology.orchestra.run.smithi112.stdout:Downloading Packages: 2024-04-25T04:05:50.215 INFO:teuthology.orchestra.run.smithi112.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.8 MB/s | 255 kB 00:00 2024-04-25T04:05:50.216 INFO:teuthology.orchestra.run.smithi112.stdout:-------------------------------------------------------------------------------- 2024-04-25T04:05:50.216 INFO:teuthology.orchestra.run.smithi112.stdout:Total 1.1 MB/s | 255 kB 00:00 2024-04-25T04:05:50.231 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction check 2024-04-25T04:05:50.414 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction check succeeded. 2024-04-25T04:05:50.415 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction test 2024-04-25T04:05:50.419 INFO:teuthology.orchestra.run.smithi112.stdout:Transaction test succeeded. 2024-04-25T04:05:50.420 INFO:teuthology.orchestra.run.smithi112.stdout:Running transaction 2024-04-25T04:05:50.548 INFO:teuthology.orchestra.run.smithi112.stdout: Preparing : 1/1 2024-04-25T04:05:50.633 INFO:teuthology.orchestra.run.smithi112.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-25T04:05:50.633 INFO:teuthology.orchestra.run.smithi112.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T04:05:51.331 INFO:teuthology.orchestra.run.smithi112.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T04:05:51.331 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-25T04:05:51.567 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-25T04:05:52.037 INFO:teuthology.orchestra.run.smithi112.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T04:05:52.037 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:05:52.037 INFO:teuthology.orchestra.run.smithi112.stdout:Reinstalled: 2024-04-25T04:05:52.037 INFO:teuthology.orchestra.run.smithi112.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-25T04:05:52.037 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:05:52.038 INFO:teuthology.orchestra.run.smithi112.stdout:Complete! 2024-04-25T04:05:52.220 DEBUG:teuthology.orchestra.run.smithi112:> rpm -q kernel | sort -rV | head -n 1 2024-04-25T04:05:52.267 INFO:teuthology.orchestra.run.smithi112.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-25T04:05:52.267 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-25T04:05:52.267 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-25T04:05:52.267 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-25T04:05:52.267 DEBUG:teuthology.orchestra.run.smithi112:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-25T04:05:52.300 INFO:teuthology.orchestra.run.smithi112.stdout:ttyS1 2024-04-25T04:05:52.320 DEBUG:teuthology.parallel:result is None 2024-04-25T04:05:53.431 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-25T04:05:53.431 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-25T04:05:53.504 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-04-25T04:05:53.505 INFO:teuthology.orchestra.run.smithi078.stdout:Nothing to do. 2024-04-25T04:05:53.505 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-04-25T04:05:53.616 DEBUG:teuthology.orchestra.run.smithi078:> echo no | sudo yum reinstall kernel || true 2024-04-25T04:05:54.114 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:03 ago on Thu 25 Apr 2024 04:05:51 AM UTC. 2024-04-25T04:05:54.234 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-04-25T04:05:54.251 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-04-25T04:05:54.251 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout:Reinstalling: 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 255 k 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 0 2024-04-25T04:05:54.252 INFO:teuthology.orchestra.run.smithi078.stderr:Operation aborted. 2024-04-25T04:05:54.309 INFO:teuthology.orchestra.run.smithi078.stdout:Is this ok [y/N]: 2024-04-25T04:05:54.309 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum reinstall -y kernel || true 2024-04-25T04:05:54.797 INFO:teuthology.orchestra.run.smithi078.stdout:Last metadata expiration check: 0:00:03 ago on Thu 25 Apr 2024 04:05:51 AM UTC. 2024-04-25T04:05:54.913 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-04-25T04:05:54.914 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-04-25T04:05:54.914 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-04-25T04:05:54.914 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-04-25T04:05:54.914 INFO:teuthology.orchestra.run.smithi078.stdout:Reinstalling: 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 255 k 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 0 2024-04-25T04:05:54.915 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-04-25T04:05:55.277 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.1 MB/s | 255 kB 00:00 2024-04-25T04:05:55.278 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-04-25T04:05:55.278 INFO:teuthology.orchestra.run.smithi078.stdout:Total 703 kB/s | 255 kB 00:00 2024-04-25T04:05:55.294 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-04-25T04:05:55.481 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-04-25T04:05:55.481 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-04-25T04:05:55.486 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-04-25T04:05:55.486 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-04-25T04:05:55.625 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-04-25T04:05:55.695 INFO:teuthology.orchestra.run.smithi078.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-25T04:05:55.695 INFO:teuthology.orchestra.run.smithi078.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T04:05:56.356 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T04:05:56.356 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-25T04:05:57.017 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-25T04:05:57.017 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:05:57.017 INFO:teuthology.orchestra.run.smithi078.stdout:Reinstalled: 2024-04-25T04:05:57.017 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-25T04:05:57.017 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:05:57.017 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-04-25T04:05:57.208 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-04-25T04:05:57.254 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-25T04:05:57.255 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-25T04:05:57.255 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-25T04:05:57.255 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-25T04:05:57.255 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-25T04:05:57.287 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-04-25T04:05:57.311 DEBUG:teuthology.parallel:result is None 2024-04-25T04:05:57.311 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-25T04:05:57.318 INFO:teuthology.task.internal:Creating test directory... 2024-04-25T04:05:57.318 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-25T04:05:57.354 DEBUG:teuthology.orchestra.run.smithi112:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-25T04:05:57.372 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-25T04:05:57.378 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-25T04:05:57.384 INFO:teuthology.task.internal:Creating archive directory... 2024-04-25T04:05:57.385 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-25T04:05:57.409 DEBUG:teuthology.orchestra.run.smithi112:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-25T04:05:57.438 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-25T04:05:57.444 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-25T04:05:57.444 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-25T04:05:57.475 DEBUG:teuthology.orchestra.run.smithi112:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-25T04:05:57.499 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T04:05:57.508 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T04:05:57.509 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T04:05:57.520 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-25T04:05:57.522 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-25T04:05:57.529 INFO:teuthology.task.internal:Configuring sudo... 2024-04-25T04:05:57.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-25T04:05:57.552 DEBUG:teuthology.orchestra.run.smithi112:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-25T04:05:57.588 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-25T04:05:57.596 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-25T04:05:57.596 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-25T04:05:57.617 DEBUG:teuthology.orchestra.run.smithi112:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-25T04:05:57.644 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T04:05:57.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T04:05:57.765 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T04:05:57.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T04:05:57.886 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:05:57.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-25T04:05:57.949 DEBUG:teuthology.orchestra.run.smithi112:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T04:05:57.993 DEBUG:teuthology.orchestra.run.smithi112:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-25T04:05:58.041 DEBUG:teuthology.orchestra.run.smithi112:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T04:05:58.127 DEBUG:teuthology.orchestra.run.smithi112:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-25T04:05:58.167 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-25T04:05:58.167 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-25T04:05:58.233 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-04-25T04:05:58.235 DEBUG:teuthology.orchestra.run.smithi112:> sudo service rsyslog restart 2024-04-25T04:05:58.266 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T04:05:58.314 INFO:teuthology.orchestra.run.smithi112.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T04:05:58.685 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-25T04:05:58.693 INFO:teuthology.task.internal:Starting timer... 2024-04-25T04:05:58.693 INFO:teuthology.run_tasks:Running task pcp... 2024-04-25T04:05:58.712 INFO:teuthology.run_tasks:Running task selinux... 2024-04-25T04:05:58.720 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-25T04:05:58.720 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-04-25T04:05:58.771 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-04-25T04:05:58.773 DEBUG:teuthology.orchestra.run.smithi112:> sudo service auditd rotate 2024-04-25T04:05:58.831 INFO:teuthology.orchestra.run.smithi112.stdout:Rotating logs: 2024-04-25T04:05:58.832 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-25T04:05:58.832 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-04-25T04:05:58.853 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-04-25T04:05:58.853 DEBUG:teuthology.orchestra.run.smithi112:> /usr/sbin/getenforce 2024-04-25T04:05:58.873 INFO:teuthology.orchestra.run.smithi112.stdout:Permissive 2024-04-25T04:05:58.873 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi078.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi112.front.sepia.ceph.com': 'permissive'} 2024-04-25T04:05:58.874 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T04:05:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:05:58.918 DEBUG:teuthology.orchestra.run.smithi112:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T04:05:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:05:58.943 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-25T04:05:58.943 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-04-25T04:05:58.989 DEBUG:teuthology.orchestra.run.smithi112:> sudo /usr/sbin/setenforce permissive 2024-04-25T04:05:59.013 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-25T04:05:59.023 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-25T04:05:59.038 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-25T04:05:59.039 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com,smithi112.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-25T04:09:25.137 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com'), Remote(name='ubuntu@smithi112.front.sepia.ceph.com')] 2024-04-25T04:09:25.138 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-04-25T04:09:25.139 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T04:09:25.212 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-04-25T04:09:25.285 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-04-25T04:09:25.286 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi112.front.sepia.ceph.com' 2024-04-25T04:09:25.286 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi112.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-25T04:09:25.362 DEBUG:teuthology.orchestra.run.smithi112:> true 2024-04-25T04:09:25.398 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi112.front.sepia.ceph.com' 2024-04-25T04:09:25.399 INFO:teuthology.run_tasks:Running task clock... 2024-04-25T04:09:25.409 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-25T04:09:25.409 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-25T04:09:25.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T04:09:25.413 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-25T04:09:25.413 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T04:09:25.438 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-25T04:09:25.451 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-25T04:09:25.468 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-25T04:09:25.473 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-04-25T04:09:25.481 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-25T04:09:25.483 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-04-25T04:09:25.496 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-25T04:09:25.503 INFO:teuthology.orchestra.run.smithi112.stderr:sudo: ntpd: command not found 2024-04-25T04:09:25.509 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-25T04:09:25.514 INFO:teuthology.orchestra.run.smithi112.stdout:506 Cannot talk to daemon 2024-04-25T04:09:25.527 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-25T04:09:25.540 INFO:teuthology.orchestra.run.smithi112.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-25T04:09:25.564 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-04-25T04:09:25.568 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T04:09:25.568 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-04-25T04:09:25.568 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.568 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.568 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.568 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.591 INFO:teuthology.orchestra.run.smithi112.stderr:bash: line 1: ntpq: command not found 2024-04-25T04:09:25.594 INFO:teuthology.orchestra.run.smithi112.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T04:09:25.594 INFO:teuthology.orchestra.run.smithi112.stdout:=============================================================================== 2024-04-25T04:09:25.594 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.594 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.595 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.595 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:09:25.595 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-25T04:09:25.694 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': 'd6f5f4e5ec08db21b2896cb71b4cb907b7fb4737'} 2024-04-25T04:09:25.694 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-25T04:09:25.695 INFO:tasks.cephadm:Cluster fsid is 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:09:25.695 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-25T04:09:25.695 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-25T04:09:25.695 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi078': '172.21.15.78', 'mon.smithi112': '172.21.15.112'} 2024-04-25T04:09:25.695 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-25T04:09:25.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo hostname $(hostname -s) 2024-04-25T04:09:25.720 DEBUG:teuthology.orchestra.run.smithi112:> sudo hostname $(hostname -s) 2024-04-25T04:09:25.743 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-25T04:09:25.744 DEBUG:teuthology.orchestra.run.smithi078:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-25T04:09:26.157 INFO:teuthology.orchestra.run.smithi078.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 25 04:09 /home/ubuntu/cephtest/cephadm 2024-04-25T04:09:26.157 DEBUG:teuthology.orchestra.run.smithi112:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-25T04:09:26.421 INFO:teuthology.orchestra.run.smithi112.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 25 04:09 /home/ubuntu/cephtest/cephadm 2024-04-25T04:09:26.421 DEBUG:teuthology.orchestra.run.smithi078:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-25T04:09:26.438 DEBUG:teuthology.orchestra.run.smithi112:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-25T04:09:26.463 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-25T04:09:26.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-25T04:09:26.481 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-25T04:09:27.929 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-25T04:09:28.060 INFO:teuthology.orchestra.run.smithi112.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-25T04:09:53.015 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:09:53.015 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-25T04:09:53.015 INFO:teuthology.orchestra.run.smithi078.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-25T04:09:53.015 INFO:teuthology.orchestra.run.smithi078.stdout: "repo_digests": [ 2024-04-25T04:09:53.015 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-25T04:09:53.015 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-25T04:09:53.016 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-04-25T04:09:53.016 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:09:53.487 INFO:teuthology.orchestra.run.smithi112.stdout:{ 2024-04-25T04:09:53.487 INFO:teuthology.orchestra.run.smithi112.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-25T04:09:53.488 INFO:teuthology.orchestra.run.smithi112.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-25T04:09:53.488 INFO:teuthology.orchestra.run.smithi112.stdout: "repo_digests": [ 2024-04-25T04:09:53.488 INFO:teuthology.orchestra.run.smithi112.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-25T04:09:53.488 INFO:teuthology.orchestra.run.smithi112.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-25T04:09:53.488 INFO:teuthology.orchestra.run.smithi112.stdout: ] 2024-04-25T04:09:53.488 INFO:teuthology.orchestra.run.smithi112.stdout:} 2024-04-25T04:09:53.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph 2024-04-25T04:09:53.534 DEBUG:teuthology.orchestra.run.smithi112:> sudo mkdir -p /etc/ceph 2024-04-25T04:09:53.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /etc/ceph 2024-04-25T04:09:53.598 DEBUG:teuthology.orchestra.run.smithi112:> sudo chmod 777 /etc/ceph 2024-04-25T04:09:53.626 INFO:tasks.cephadm:Writing seed config... 2024-04-25T04:09:53.627 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-25T04:09:53.627 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-25T04:09:53.628 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-25T04:09:53.628 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-25T04:09:53.628 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-25T04:09:53.628 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-25T04:09:53.628 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-25T04:09:53.628 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-25T04:09:53.628 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:09:53.629 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-25T04:09:53.652 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9a8d9f72-02b9-11ef-bc93-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-25T04:09:53.652 DEBUG:teuthology.orchestra.run.smithi078:mon.smithi078> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service 2024-04-25T04:09:53.695 INFO:tasks.cephadm:Bootstrapping... 2024-04-25T04:09:53.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.78 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-25T04:09:53.886 INFO:teuthology.orchestra.run.smithi078.stderr:-------------------------------------------------------------------------------- 2024-04-25T04:09:53.886 INFO:teuthology.orchestra.run.smithi078.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '9a8d9f72-02b9-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.78', '--skip-admin-label'] 2024-04-25T04:09:53.909 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 4.9.4-dev 2024-04-25T04:09:53.911 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying podman|docker is present... 2024-04-25T04:09:53.932 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 4.9.4-dev 2024-04-25T04:09:53.935 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying lvm2 is present... 2024-04-25T04:09:53.935 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying time synchronization is in place... 2024-04-25T04:09:53.942 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-25T04:09:53.948 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-04-25T04:09:53.955 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-04-25T04:09:53.961 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-04-25T04:09:53.961 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-04-25T04:09:53.961 INFO:teuthology.orchestra.run.smithi078.stderr:Repeating the final host check... 2024-04-25T04:09:53.983 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 4.9.4-dev 2024-04-25T04:09:53.985 INFO:teuthology.orchestra.run.smithi078.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-25T04:09:53.985 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl is present 2024-04-25T04:09:53.985 INFO:teuthology.orchestra.run.smithi078.stderr:lvcreate is present 2024-04-25T04:09:53.991 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-25T04:09:53.997 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-04-25T04:09:54.003 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-04-25T04:09:54.010 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-04-25T04:09:54.010 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-04-25T04:09:54.010 INFO:teuthology.orchestra.run.smithi078.stderr:Host looks OK 2024-04-25T04:09:54.010 INFO:teuthology.orchestra.run.smithi078.stderr:Cluster fsid: 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:09:54.011 INFO:teuthology.orchestra.run.smithi078.stderr:Acquiring lock 139974527366432 on /run/cephadm/9a8d9f72-02b9-11ef-bc93-c7b262605968.lock 2024-04-25T04:09:54.011 INFO:teuthology.orchestra.run.smithi078.stderr:Lock 139974527366432 acquired on /run/cephadm/9a8d9f72-02b9-11ef-bc93-c7b262605968.lock 2024-04-25T04:09:54.011 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 3300 ... 2024-04-25T04:09:54.011 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 6789 ... 2024-04-25T04:09:54.011 INFO:teuthology.orchestra.run.smithi078.stderr:Base mon IP is 172.21.15.78, final addrv is [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-04-25T04:09:54.014 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.78 metric 100 2024-04-25T04:09:54.014 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.78 metric 100 2024-04-25T04:09:54.016 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-25T04:09:54.016 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-25T04:09:54.017 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-25T04:09:54.019 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-25T04:09:54.019 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-25T04:09:54.019 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-25T04:09:54.019 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-25T04:09:54.019 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 fe80::b0fa:2a53:aed7:8bd1/64 scope link noprefixroute 2024-04-25T04:09:54.019 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-25T04:09:54.020 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.0.0/20` 2024-04-25T04:09:54.020 INFO:teuthology.orchestra.run.smithi078.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-25T04:09:54.021 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-25T04:09:54.123 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-25T04:09:54.583 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Getting image source signatures 2024-04-25T04:09:54.583 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-25T04:09:54.583 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-25T04:09:54.583 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-25T04:09:54.583 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-25T04:09:54.583 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-25T04:09:54.583 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-25T04:09:54.751 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Writing manifest to image destination 2024-04-25T04:09:54.913 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-25T04:09:55.294 INFO:teuthology.orchestra.run.smithi078.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-25T04:09:55.844 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-25T04:09:55.845 INFO:teuthology.orchestra.run.smithi078.stderr:Extracting ceph user uid/gid from container image... 2024-04-25T04:09:56.279 INFO:teuthology.orchestra.run.smithi078.stderr:stat: 167 167 2024-04-25T04:09:56.759 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial keys... 2024-04-25T04:09:57.126 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQCV1ylmfSM/AhAAaNat1fhBMsQRGYPeGt6cLg== 2024-04-25T04:09:58.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQCV1ylm3q9HORAAX9Q67tbvWOUsCN81KnNLZw== 2024-04-25T04:10:01.673 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQCY1ylmwr8iJhAAb+5X5X5347a56RFAp+3ADg== 2024-04-25T04:10:07.210 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial monmap... 2024-04-25T04:10:09.678 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-25T04:10:09.678 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-25T04:10:09.678 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:09.678 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-25T04:10:10.142 INFO:teuthology.orchestra.run.smithi078.stderr:monmaptool for smithi078 [v2:172.21.15.78:3300,v1:172.21.15.78:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-25T04:10:10.142 INFO:teuthology.orchestra.run.smithi078.stderr:setting min_mon_release = octopus 2024-04-25T04:10:10.142 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: set fsid to 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:10.142 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-25T04:10:10.142 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:10:10.142 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mon... 2024-04-25T04:10:11.057 INFO:teuthology.orchestra.run.smithi078.stderr:create mon.smithi078 on 2024-04-25T04:10:11.262 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-25T04:10:11.411 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968.target. 2024-04-25T04:10:11.411 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968.target. 2024-04-25T04:10:11.701 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service: Unit ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service not loaded. 2024-04-25T04:10:11.708 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968.target.wants/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service → /etc/systemd/system/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@.service. 2024-04-25T04:10:12.598 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-04-25T04:10:12.604 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-04-25T04:10:12.605 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-04-25T04:10:12.605 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-04-25T04:10:12.606 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon to start... 2024-04-25T04:10:12.606 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon... 2024-04-25T04:10:12.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:12 smithi078 systemd[1]: Started Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:10:13.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: cluster: 2024-04-25T04:10:13.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: id: 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:13.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-25T04:10:13.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:13.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: services: 2024-04-25T04:10:13.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi078 (age 0.589184s) 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: data: 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pgs: 2024-04-25T04:10:13.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:13 smithi078 ceph-mon[21469]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-04-25T04:10:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:13 smithi078 ceph-mon[21469]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-04-25T04:10:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:13 smithi078 ceph-mon[21469]: fsmap 2024-04-25T04:10:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:13 smithi078 ceph-mon[21469]: osdmap e1: 0 total, 0 up, 0 in 2024-04-25T04:10:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:13 smithi078 ceph-mon[21469]: mgrmap e1: no daemons active 2024-04-25T04:10:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:13 smithi078 ceph-mon[21469]: from='client.? 172.21.15.78:0/454658736' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-25T04:10:13.736 INFO:teuthology.orchestra.run.smithi078.stderr:mon is available 2024-04-25T04:10:13.736 INFO:teuthology.orchestra.run.smithi078.stderr:Assimilating anything we can from ceph.conf... 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-04-25T04:10:14.253 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-25T04:10:14.254 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:14.254 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-04-25T04:10:14.254 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-25T04:10:14.254 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-25T04:10:14.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:14 smithi078 ceph-mon[21469]: from='client.? 172.21.15.78:0/288440660' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-25T04:10:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:14 smithi078 ceph-mon[21469]: from='client.? 172.21.15.78:0/288440660' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-25T04:10:15.442 INFO:teuthology.orchestra.run.smithi078.stderr:Generating new minimal ceph.conf... 2024-04-25T04:10:16.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:15 smithi078 ceph-mon[21469]: from='client.? 172.21.15.78:0/1715748793' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:10:16.474 INFO:teuthology.orchestra.run.smithi078.stderr:Restarting the monitor... 2024-04-25T04:10:16.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:16 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:10:16.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:16 smithi078 bash[21699]: Error: no container with name or ID "ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon.smithi078" found: no such container 2024-04-25T04:10:16.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:16 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[21465]: 2024-04-25T04:10:16.652+0000 7f8cf91ad700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:10:16.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:16 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[21465]: 2024-04-25T04:10:16.652+0000 7f8cf91ad700 -1 mon.smithi078@0(leader) e1 *** Got Signal Terminated *** 2024-04-25T04:10:17.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:16 smithi078 podman[21712]: 2024-04-25 04:10:16.918893753 +0000 UTC m=+0.393688009 container died 9ac505ef725ee828e27d0a565fc54575c3c30b1444c332514405369223595f6a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, vcs-type=git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-04-25T04:10:17.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 podman[21712]: 2024-04-25 04:10:17.252522736 +0000 UTC m=+0.727316990 container cleanup 9ac505ef725ee828e27d0a565fc54575c3c30b1444c332514405369223595f6a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, name=centos-stream, distribution-scope=public, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-25T04:10:17.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 bash[21712]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078 2024-04-25T04:10:17.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 podman[21726]: 2024-04-25 04:10:17.511724642 +0000 UTC m=+0.592601604 container remove 9ac505ef725ee828e27d0a565fc54575c3c30b1444c332514405369223595f6a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, version=8, GIT_CLEAN=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.openshift.expose-services=, com.redhat.component=centos-stream-container, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8) 2024-04-25T04:10:17.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 bash[21739]: Error: no container with name or ID "ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon.smithi078" found: no such container 2024-04-25T04:10:17.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-04-25T04:10:17.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:10:17.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:10:18.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 podman[21810]: 2024-04-25 04:10:17.673225508 +0000 UTC m=+0.018642958 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-25T04:10:18.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:17 smithi078 podman[21810]: 2024-04-25 04:10:17.787337067 +0000 UTC m=+0.132754562 container create 74117070b5c0cc237da0cfed9ae6c06df00bc650df578d92aa6f0d3d1028ff92 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_CLEAN=True, GIT_BRANCH=HEAD, release=754, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, RELEASE=HEAD, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0) 2024-04-25T04:10:18.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 podman[21810]: 2024-04-25 04:10:18.032268733 +0000 UTC m=+0.377686193 container init 74117070b5c0cc237da0cfed9ae6c06df00bc650df578d92aa6f0d3d1028ff92 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.expose-services=, vendor=Red Hat, Inc., ceph=True, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True) 2024-04-25T04:10:18.122 INFO:teuthology.orchestra.run.smithi078.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 podman[21810]: 2024-04-25 04:10:18.035600615 +0000 UTC m=+0.381018068 container start 74117070b5c0cc237da0cfed9ae6c06df00bc650df578d92aa6f0d3d1028ff92 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, release=754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64) 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: set uid:gid to 167:167 (ceph:ceph) 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: pidfile_write: ignore empty --pid-file 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: load: jerasure load: lrc 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: RocksDB version: 6.15.5 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Compile date Apr 18 2022 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: DB SUMMARY 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: DB Session ID: VTJIGI4UQZ39K4BKGX4Q 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: CURRENT file: CURRENT 2024-04-25T04:10:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: IDENTITY file: IDENTITY 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000008.sst 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000010.log size: 82701 ; 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.error_if_exists: 0 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.create_if_missing: 0 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.paranoid_checks: 1 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.env: 0x564ec44c6860 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.fs: Posix File System 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.info_log: 0x564ec6611fa0 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_file_opening_threads: 16 2024-04-25T04:10:18.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.statistics: (nil) 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.use_fsync: 0 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_log_file_size: 0 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.keep_log_file_num: 1000 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.recycle_log_file_num: 0 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.allow_fallocate: 1 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.allow_mmap_reads: 0 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.allow_mmap_writes: 0 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.use_direct_reads: 0 2024-04-25T04:10:18.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.create_missing_column_families: 0 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.db_log_dir: 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi078/store.db 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.advise_random_on_open: 1 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.db_write_buffer_size: 0 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.write_buffer_manager: 0x564ec6702330 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-25T04:10:18.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.rate_limiter: (nil) 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.wal_recovery_mode: 2 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.enable_thread_tracking: 0 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.enable_pipelined_write: 0 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.unordered_write: 0 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.row_cache: None 2024-04-25T04:10:18.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.wal_filter: None 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.allow_ingest_behind: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.preserve_deletes: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.two_write_queues: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.manual_wal_flush: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.atomic_flush: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.log_readahead_size: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.best_efforts_recovery: 0 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-25T04:10:18.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.allow_data_in_errors: 0 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.db_host_id: __hostname__ 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_background_jobs: 2 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_background_compactions: -1 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_subcompactions: 1 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_total_wal_size: 0 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-25T04:10:18.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_open_files: -1 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bytes_per_sync: 0 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_readahead_size: 0 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_background_flushes: -1 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Compression algorithms supported: 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kZSTD supported: 0 2024-04-25T04:10:18.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kXpressCompression supported: 0 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kLZ4HCCompression supported: 1 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kLZ4Compression supported: 1 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kBZip2Compression supported: 0 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kZlibCompression supported: 1 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: kSnappyCompression supported: 1 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.merge_operator: 2024-04-25T04:10:18.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_filter: None 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_filter_factory: None 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.sst_partitioner_factory: None 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564ec65ddd00) 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-04-25T04:10:18.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: hash_index_allow_collision: 1 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 1 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x564ec6648170 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-04-25T04:10:18.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-04-25T04:10:18.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 4 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.write_buffer_size: 33554432 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_write_buffer_number: 2 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression: NoCompression 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression: Disabled 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.prefix_extractor: nullptr 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.num_levels: 7 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-25T04:10:18.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression_opts.level: 32767 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression_opts.strategy: 0 2024-04-25T04:10:18.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compression_opts.enabled: false 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.target_file_size_base: 67108864 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-25T04:10:18.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.arena_block_size: 4194304 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-25T04:10:18.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.disable_auto_compactions: 0 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-25T04:10:18.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.table_properties_collectors: 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.inplace_update_support: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.bloom_locality: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.max_successive_merges: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.paranoid_file_checks: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.force_consistency_checks: 1 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.report_bg_io_stats: 0 2024-04-25T04:10:18.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.ttl: 2592000 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.enable_blob_files: false 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.min_blob_size: 0 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.blob_file_size: 268435456 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018218061519, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-25T04:10:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018218062281, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79383, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77647, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71956, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714018218, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "6fcc9ba9-da85-472e-baba-c561b807e90e", "db_session_id": "VTJIGI4UQZ39K4BKGX4Q"}} 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018218062610, "job": 1, "event": "recovery_finished"} 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x564ec662ea80 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: DB pointer 0x564ec663e000 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-25T04:10:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:10:18.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 25.68 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 25.68 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-25T04:10:18.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-25T04:10:18.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 139.4 0.00 0.00 1 0.001 0 0 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 25.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-25T04:10:18.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???) e1 preinit fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???).mds e1 new map 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???).mds e1 print_map 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e1 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: -1 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: No filesystems configured 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T04:10:18.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expand map: {default=false} 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta from 'false' to 'false' 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expanded map: {default=false} 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expand map: {default=info} 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta from 'info' to 'info' 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expanded map: {default=info} 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expand map: {default=daemon} 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expanded map: {default=daemon} 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expand map: {default=debug} 2024-04-25T04:10:18.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta from 'debug' to 'debug' 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: expand_channel_meta expanded map: {default=debug} 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 bash[21810]: 74117070b5c0cc237da0cfed9ae6c06df00bc650df578d92aa6f0d3d1028ff92 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 systemd[1]: Started Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: fsmap 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: osdmap e1: 0 total, 0 up, 0 in 2024-04-25T04:10:18.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:18 smithi078 ceph-mon[21828]: mgrmap e1: no daemons active 2024-04-25T04:10:19.236 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-25T04:10:19.236 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-25T04:10:19.236 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mgr... 2024-04-25T04:10:19.236 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying port 9283 ... 2024-04-25T04:10:19.399 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mgr.smithi078.eugwik.service: Unit ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mgr.smithi078.eugwik.service not loaded. 2024-04-25T04:10:19.406 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968.target.wants/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mgr.smithi078.eugwik.service → /etc/systemd/system/ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@.service. 2024-04-25T04:10:19.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:19 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3736493810' entity='client.admin' 2024-04-25T04:10:20.400 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-04-25T04:10:20.407 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-04-25T04:10:20.409 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-04-25T04:10:20.409 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-04-25T04:10:20.416 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-04-25T04:10:20.423 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-04-25T04:10:20.424 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-04-25T04:10:20.424 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-25T04:10:20.425 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr to start... 2024-04-25T04:10:20.425 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr... 2024-04-25T04:10:20.946 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:20.946 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:10:20.946 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "9a8d9f72-02b9-11ef-bc93-c7b262605968", 2024-04-25T04:10:20.946 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-04-25T04:10:20.946 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-25T04:10:20.946 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-25T04:10:20.947 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-25T04:10:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-25T04:10:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-04-25T04:10:12.540402+0000", 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-25T04:10:20.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:10:21.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:20 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2448388485' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-25T04:10:21.538 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (1/15)... 2024-04-25T04:10:24.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:24.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:10:24.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "9a8d9f72-02b9-11ef-bc93-c7b262605968", 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-04-25T04:10:24.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-25T04:10:24.081 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-25T04:10:24.086 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-04-25T04:10:24.087 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-04-25T04:10:12.540402+0000", 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-25T04:10:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:10:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:24 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1091370758' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-25T04:10:24.719 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (2/15)... 2024-04-25T04:10:26.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: Activating manager daemon smithi078.eugwik 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: mgrmap e2: smithi078.eugwik(active, starting, since 0.00256519s) 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:26 smithi078 ceph-mon[21828]: from='mgr.14100 172.21.15.78:0/2460257846' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:27.234 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:27.285 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:10:27.285 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "9a8d9f72-02b9-11ef-bc93-c7b262605968", 2024-04-25T04:10:27.285 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-04-25T04:10:27.285 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-25T04:10:27.285 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-04-25T04:10:27.286 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:27.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-25T04:10:27.288 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:27.289 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-04-25T04:10:27.290 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:27.290 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-04-25T04:10:12.540402+0000", 2024-04-25T04:10:27.290 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:27.290 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:27.290 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-25T04:10:27.290 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:10:27.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:27 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2718224811' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-25T04:10:27.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:27 smithi078 ceph-mon[21828]: mgrmap e3: smithi078.eugwik(active, since 1.00532s) 2024-04-25T04:10:27.825 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (3/15)... 2024-04-25T04:10:30.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:30 smithi078 ceph-mon[21828]: mgrmap e4: smithi078.eugwik(active, since 2s) 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "9a8d9f72-02b9-11ef-bc93-c7b262605968", 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-04-25T04:10:30.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-25T04:10:30.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-25T04:10:30.548 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-04-25T04:10:30.549 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-04-25T04:10:12.540402+0000", 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-25T04:10:30.550 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:10:31.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:31 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2257831859' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-25T04:10:31.232 INFO:teuthology.orchestra.run.smithi078.stderr:mgr is available 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-25T04:10:31.826 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:31.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-04-25T04:10:31.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-25T04:10:31.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-04-25T04:10:31.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-04-25T04:10:31.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-25T04:10:31.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-25T04:10:32.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:32 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/931582588' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-25T04:10:32.395 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling cephadm module... 2024-04-25T04:10:33.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:33 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1278089659' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-25T04:10:34.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:34 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1278089659' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-25T04:10:34.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:34 smithi078 ceph-mon[21828]: mgrmap e5: smithi078.eugwik(active, since 6s) 2024-04-25T04:10:34.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:10:34.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-25T04:10:34.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-04-25T04:10:34.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.eugwik", 2024-04-25T04:10:34.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-25T04:10:34.340 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:10:34.900 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-04-25T04:10:34.900 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 5... 2024-04-25T04:10:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:35 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2145937158' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-25T04:10:39.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:39 smithi078 ceph-mon[21828]: Active manager daemon smithi078.eugwik restarted 2024-04-25T04:10:39.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:39 smithi078 ceph-mon[21828]: Activating manager daemon smithi078.eugwik 2024-04-25T04:10:39.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:39 smithi078 ceph-mon[21828]: osdmap e2: 0 total, 0 up, 0 in 2024-04-25T04:10:40.187 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:10:40.187 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-25T04:10:40.188 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-04-25T04:10:40.188 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:10:40.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: mgrmap e6: smithi078.eugwik(active, starting, since 0.0526159s) 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:10:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:10:40.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:10:40.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: [25/Apr/2024:04:10:39] ENGINE Bus STARTING 2024-04-25T04:10:40.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: [25/Apr/2024:04:10:40] ENGINE Serving on https://172.21.15.78:7150 2024-04-25T04:10:40.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: [25/Apr/2024:04:10:40] ENGINE Bus STARTED 2024-04-25T04:10:40.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:40.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:40 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:10:40.747 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 5 is available 2024-04-25T04:10:40.747 INFO:teuthology.orchestra.run.smithi078.stderr:Setting orchestrator backend to cephadm... 2024-04-25T04:10:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:41 smithi078 ceph-mon[21828]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-25T04:10:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:41 smithi078 ceph-mon[21828]: mgrmap e7: smithi078.eugwik(active, since 1.06045s) 2024-04-25T04:10:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:41 smithi078 ceph-mon[21828]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-25T04:10:42.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:42 smithi078 ceph-mon[21828]: mgrmap e8: smithi078.eugwik(active, since 2s) 2024-04-25T04:10:42.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:42 smithi078 ceph-mon[21828]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:42.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:42 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:42.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:42 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:10:42.585 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: value unchanged 2024-04-25T04:10:43.142 INFO:teuthology.orchestra.run.smithi078.stderr:Generating ssh key... 2024-04-25T04:10:44.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:44 smithi078 ceph-mon[21828]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:44.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:44 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:44.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:44 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:45.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:45 smithi078 ceph-mon[21828]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:45.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:45 smithi078 ceph-mon[21828]: Generating ssh key... 2024-04-25T04:10:45.227 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyyQTypjH1ydxxbBOFpN64CzPNIZerrtpOsDSN+kG5S04Bcf2iR4nhH5ydDaqJfTK0IlEDu21+SgNtUhWQN+2RLtDoJTbcr3myRtefdFd7L7Y1RukYPnedg3Rj29ZHEwXYyCOkvEmbwrqiUwYi93DcmvThhGU9NrWSfdVzgvbjwFgbam6/SGNQoa92XZJY26Nea/5COt7ZgPofxjxQVE7TMUv4FzYYRM7VOAr6+JiHoaRy9FZj+52OKd4SqJ0DElR2U3fMTwLlKD7D+B/Dx/HF0E6SgwSZIZYHiqaWwqf37Rnn2OnBSfGQvNkDdJhw/OSBonHf1s2UrYP0VOeLrB5rYpNp7d4eMUEckuX5UqO2P0jZy+QL95/B6LSXN3HT+ollhULO4tImZLS/EvnGkZK9NThhoJkMVwv0r7U4YWedXVA/V7pQe3fJTKUNEanwIxueK5gm2AV7tcFY4T7eSWnrIma+Ch5PAQ/Diyq7CFWvrm5YyYGUHIyPnOc+grZA6BU= ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:10:45.830 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-25T04:10:45.830 INFO:teuthology.orchestra.run.smithi078.stderr:Adding key to root@localhost authorized_keys... 2024-04-25T04:10:45.831 INFO:teuthology.orchestra.run.smithi078.stderr:Adding host smithi078... 2024-04-25T04:10:46.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:46 smithi078 ceph-mon[21828]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:47.301 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Added host 'smithi078' with addr '172.21.15.78' 2024-04-25T04:10:47.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:47 smithi078 ceph-mon[21828]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "addr": "172.21.15.78", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:47.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:47 smithi078 ceph-mon[21828]: Deploying cephadm binary to smithi078 2024-04-25T04:10:47.878 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mon service with default placement... 2024-04-25T04:10:48.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:48 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:48.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:48 smithi078 ceph-mon[21828]: Added host smithi078 2024-04-25T04:10:48.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:48 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:10:48.528 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-25T04:10:49.202 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mgr service with default placement... 2024-04-25T04:10:49.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:49 smithi078 ceph-mon[21828]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:49.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:49 smithi078 ceph-mon[21828]: Saving service mon spec with placement count:5 2024-04-25T04:10:49.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:49 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:50.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-25T04:10:51.257 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:51 smithi078 ceph-mon[21828]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:51.257 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:51 smithi078 ceph-mon[21828]: Saving service mgr spec with placement count:2 2024-04-25T04:10:51.257 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:51 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:51.648 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying crash service with default placement... 2024-04-25T04:10:52.284 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-25T04:10:52.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:52 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:52.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:52 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:53.111 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying prometheus service with default placement... 2024-04-25T04:10:53.411 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:53 smithi078 ceph-mon[21828]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:53.411 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:53 smithi078 ceph-mon[21828]: Saving service crash spec with placement * 2024-04-25T04:10:53.411 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:53 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:53.699 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-25T04:10:54.460 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying grafana service with default placement... 2024-04-25T04:10:54.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:54 smithi078 ceph-mon[21828]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:54.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:54 smithi078 ceph-mon[21828]: Saving service prometheus spec with placement count:1 2024-04-25T04:10:54.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:54 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:55.089 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-25T04:10:55.659 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying node-exporter service with default placement... 2024-04-25T04:10:56.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:55 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:56.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:55 smithi078 ceph-mon[21828]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:56.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:55 smithi078 ceph-mon[21828]: Saving service grafana spec with placement count:1 2024-04-25T04:10:56.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:55 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:56.489 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-25T04:10:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:57 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:57 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:57.150 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying alertmanager service with default placement... 2024-04-25T04:10:57.792 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-25T04:10:58.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:58 smithi078 ceph-mon[21828]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:58.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:58 smithi078 ceph-mon[21828]: Saving service node-exporter spec with placement * 2024-04-25T04:10:58.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:58 smithi078 ceph-mon[21828]: from='mgr.14120 172.21.15.78:0/2754102696' entity='mgr.smithi078.eugwik' 2024-04-25T04:10:59.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:59 smithi078 ceph-mon[21828]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:10:59.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:10:59 smithi078 ceph-mon[21828]: Saving service alertmanager spec with placement count:1 2024-04-25T04:11:00.384 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:00 smithi078 ceph-mon[21828]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:00.384 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:00 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/137216192' entity='client.admin' 2024-04-25T04:11:01.449 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling the dashboard module... 2024-04-25T04:11:02.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:01 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2588534926' entity='client.admin' 2024-04-25T04:11:03.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:02 smithi078 ceph-mon[21828]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:03.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:02 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2202001363' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-25T04:11:04.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:03 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2202001363' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-25T04:11:04.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:03 smithi078 ceph-mon[21828]: mgrmap e9: smithi078.eugwik(active, since 23s) 2024-04-25T04:11:04.241 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:11:04.241 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-25T04:11:04.241 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-04-25T04:11:04.241 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.eugwik", 2024-04-25T04:11:04.241 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-25T04:11:04.241 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:11:04.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:04 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2362770229' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-25T04:11:04.871 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-04-25T04:11:04.872 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 9... 2024-04-25T04:11:09.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:09 smithi078 ceph-mon[21828]: Active manager daemon smithi078.eugwik restarted 2024-04-25T04:11:09.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:09 smithi078 ceph-mon[21828]: Activating manager daemon smithi078.eugwik 2024-04-25T04:11:09.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:09 smithi078 ceph-mon[21828]: osdmap e3: 0 total, 0 up, 0 in 2024-04-25T04:11:10.102 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-04-25T04:11:10.102 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-25T04:11:10.102 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-04-25T04:11:10.102 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-04-25T04:11:10.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: mgrmap e10: smithi078.eugwik(active, starting, since 0.275666s) 2024-04-25T04:11:10.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:11:10.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:11:10.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:11:10.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: [25/Apr/2024:04:11:09] ENGINE Bus STARTING 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: [25/Apr/2024:04:11:09] ENGINE Serving on https://172.21.15.78:7150 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: [25/Apr/2024:04:11:09] ENGINE Bus STARTED 2024-04-25T04:11:10.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:10 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:10.965 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 9 is available 2024-04-25T04:11:10.965 INFO:teuthology.orchestra.run.smithi078.stderr:Generating a dashboard self-signed certificate... 2024-04-25T04:11:11.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:11 smithi078 ceph-mon[21828]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-25T04:11:11.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:11 smithi078 ceph-mon[21828]: mgrmap e11: smithi078.eugwik(active, since 1.28792s) 2024-04-25T04:11:11.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:11 smithi078 ceph-mon[21828]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-25T04:11:11.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:11 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:11.676 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-25T04:11:12.363 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial admin user... 2024-04-25T04:11:12.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:12 smithi078 ceph-mon[21828]: mgrmap e12: smithi078.eugwik(active, since 2s) 2024-04-25T04:11:12.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:12 smithi078 ceph-mon[21828]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:11:12.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:12 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:12.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:12 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:12.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:12 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:13.269 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$INCjZOJFA7bfD5tKv5GP2e1NPyoVhC7lHBHyKKS8FvXnRjR3N8xgi", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714018273, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-25T04:11:14.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:14 smithi078 ceph-mon[21828]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:11:14.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:14 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:14.770 INFO:teuthology.orchestra.run.smithi078.stderr:Fetching dashboard port number... 2024-04-25T04:11:15.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 8443 2024-04-25T04:11:16.045 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-04-25T04:11:16.052 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-04-25T04:11:16.053 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-04-25T04:11:16.053 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-25T04:11:16.055 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph Dashboard is now available at: 2024-04-25T04:11:16.055 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:16.055 INFO:teuthology.orchestra.run.smithi078.stderr: URL: https://smithi078.front.sepia.ceph.com:8443/ 2024-04-25T04:11:16.055 INFO:teuthology.orchestra.run.smithi078.stderr: User: admin 2024-04-25T04:11:16.055 INFO:teuthology.orchestra.run.smithi078.stderr: Password: ersvb8zxfc 2024-04-25T04:11:16.055 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:16.055 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling autotune for osd_memory_target 2024-04-25T04:11:16.300 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:16 smithi078 ceph-mon[21828]: mgrmap e13: smithi078.eugwik(active, since 6s) 2024-04-25T04:11:16.300 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:16 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3061319367' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-25T04:11:18.157 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-25T04:11:18.864 INFO:teuthology.orchestra.run.smithi078.stderr:You can access the Ceph CLI with: 2024-04-25T04:11:18.864 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:18.864 INFO:teuthology.orchestra.run.smithi078.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-25T04:11:18.864 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr: ceph telemetry on 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:For more information see: 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:11:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:Bootstrap complete. 2024-04-25T04:11:18.898 INFO:tasks.cephadm:Fetching config... 2024-04-25T04:11:18.899 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:11:18.899 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-25T04:11:18.914 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-25T04:11:18.914 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:11:18.914 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-25T04:11:18.970 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-25T04:11:18.970 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:11:18.970 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/keyring of=/dev/stdout 2024-04-25T04:11:19.032 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-25T04:11:19.033 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:11:19.033 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-25T04:11:19.087 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-25T04:11:19.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyyQTypjH1ydxxbBOFpN64CzPNIZerrtpOsDSN+kG5S04Bcf2iR4nhH5ydDaqJfTK0IlEDu21+SgNtUhWQN+2RLtDoJTbcr3myRtefdFd7L7Y1RukYPnedg3Rj29ZHEwXYyCOkvEmbwrqiUwYi93DcmvThhGU9NrWSfdVzgvbjwFgbam6/SGNQoa92XZJY26Nea/5COt7ZgPofxjxQVE7TMUv4FzYYRM7VOAr6+JiHoaRy9FZj+52OKd4SqJ0DElR2U3fMTwLlKD7D+B/Dx/HF0E6SgwSZIZYHiqaWwqf37Rnn2OnBSfGQvNkDdJhw/OSBonHf1s2UrYP0VOeLrB5rYpNp7d4eMUEckuX5UqO2P0jZy+QL95/B6LSXN3HT+ollhULO4tImZLS/EvnGkZK9NThhoJkMVwv0r7U4YWedXVA/V7pQe3fJTKUNEanwIxueK5gm2AV7tcFY4T7eSWnrIma+Ch5PAQ/Diyq7CFWvrm5YyYGUHIyPnOc+grZA6BU= ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-25T04:11:19.160 INFO:teuthology.orchestra.run.smithi078.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyyQTypjH1ydxxbBOFpN64CzPNIZerrtpOsDSN+kG5S04Bcf2iR4nhH5ydDaqJfTK0IlEDu21+SgNtUhWQN+2RLtDoJTbcr3myRtefdFd7L7Y1RukYPnedg3Rj29ZHEwXYyCOkvEmbwrqiUwYi93DcmvThhGU9NrWSfdVzgvbjwFgbam6/SGNQoa92XZJY26Nea/5COt7ZgPofxjxQVE7TMUv4FzYYRM7VOAr6+JiHoaRy9FZj+52OKd4SqJ0DElR2U3fMTwLlKD7D+B/Dx/HF0E6SgwSZIZYHiqaWwqf37Rnn2OnBSfGQvNkDdJhw/OSBonHf1s2UrYP0VOeLrB5rYpNp7d4eMUEckuX5UqO2P0jZy+QL95/B6LSXN3HT+ollhULO4tImZLS/EvnGkZK9NThhoJkMVwv0r7U4YWedXVA/V7pQe3fJTKUNEanwIxueK5gm2AV7tcFY4T7eSWnrIma+Ch5PAQ/Diyq7CFWvrm5YyYGUHIyPnOc+grZA6BU= ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:11:19.171 DEBUG:teuthology.orchestra.run.smithi112:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyyQTypjH1ydxxbBOFpN64CzPNIZerrtpOsDSN+kG5S04Bcf2iR4nhH5ydDaqJfTK0IlEDu21+SgNtUhWQN+2RLtDoJTbcr3myRtefdFd7L7Y1RukYPnedg3Rj29ZHEwXYyCOkvEmbwrqiUwYi93DcmvThhGU9NrWSfdVzgvbjwFgbam6/SGNQoa92XZJY26Nea/5COt7ZgPofxjxQVE7TMUv4FzYYRM7VOAr6+JiHoaRy9FZj+52OKd4SqJ0DElR2U3fMTwLlKD7D+B/Dx/HF0E6SgwSZIZYHiqaWwqf37Rnn2OnBSfGQvNkDdJhw/OSBonHf1s2UrYP0VOeLrB5rYpNp7d4eMUEckuX5UqO2P0jZy+QL95/B6LSXN3HT+ollhULO4tImZLS/EvnGkZK9NThhoJkMVwv0r7U4YWedXVA/V7pQe3fJTKUNEanwIxueK5gm2AV7tcFY4T7eSWnrIma+Ch5PAQ/Diyq7CFWvrm5YyYGUHIyPnOc+grZA6BU= ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-25T04:11:19.211 INFO:teuthology.orchestra.run.smithi112.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyyQTypjH1ydxxbBOFpN64CzPNIZerrtpOsDSN+kG5S04Bcf2iR4nhH5ydDaqJfTK0IlEDu21+SgNtUhWQN+2RLtDoJTbcr3myRtefdFd7L7Y1RukYPnedg3Rj29ZHEwXYyCOkvEmbwrqiUwYi93DcmvThhGU9NrWSfdVzgvbjwFgbam6/SGNQoa92XZJY26Nea/5COt7ZgPofxjxQVE7TMUv4FzYYRM7VOAr6+JiHoaRy9FZj+52OKd4SqJ0DElR2U3fMTwLlKD7D+B/Dx/HF0E6SgwSZIZYHiqaWwqf37Rnn2OnBSfGQvNkDdJhw/OSBonHf1s2UrYP0VOeLrB5rYpNp7d4eMUEckuX5UqO2P0jZy+QL95/B6LSXN3HT+ollhULO4tImZLS/EvnGkZK9NThhoJkMVwv0r7U4YWedXVA/V7pQe3fJTKUNEanwIxueK5gm2AV7tcFY4T7eSWnrIma+Ch5PAQ/Diyq7CFWvrm5YyYGUHIyPnOc+grZA6BU= ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:11:19.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-25T04:11:19.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:19 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1412446577' entity='client.admin' 2024-04-25T04:11:23.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:23 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:23.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:23 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:11:23.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:23 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:23.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:23 smithi078 ceph-mon[21828]: Deploying daemon alertmanager.smithi078 on smithi078 2024-04-25T04:11:23.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:23 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1078499649' entity='client.admin' 2024-04-25T04:11:23.897 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-25T04:11:23.898 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-25T04:11:28.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:28 smithi078 ceph-mon[21828]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:11:28.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:28 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:28.386 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi112 2024-04-25T04:11:28.387 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-25T04:11:28.387 DEBUG:teuthology.orchestra.run.smithi112:> dd of=/etc/ceph/ceph.conf 2024-04-25T04:11:28.405 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-25T04:11:28.405 DEBUG:teuthology.orchestra.run.smithi112:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:11:28.464 INFO:tasks.cephadm:Adding host smithi112 to orchestrator... 2024-04-25T04:11:28.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch host add smithi112 2024-04-25T04:11:30.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:30 smithi078 ceph-mon[21828]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:32.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:32 smithi078 ceph-mon[21828]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:33.939 INFO:teuthology.orchestra.run.smithi078.stdout:Added host 'smithi112' with addr '172.21.15.112' 2024-04-25T04:11:34.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-25T04:11:34.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi112", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:11:34.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: Deploying cephadm binary to smithi112 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: Deploying daemon crash.smithi078 on smithi078 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:34.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:34 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:35.696 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:35 smithi078 ceph-mon[21828]: Added host smithi112 2024-04-25T04:11:36.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:36 smithi078 ceph-mon[21828]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:37.662 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:11:37.662 INFO:teuthology.orchestra.run.smithi078.stdout:[{"addr": "172.21.15.78", "hostname": "smithi078", "labels": [], "status": ""}, {"addr": "172.21.15.112", "hostname": "smithi112", "labels": [], "status": ""}] 2024-04-25T04:11:38.663 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-25T04:11:38.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-25T04:11:38.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:38 smithi078 ceph-mon[21828]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:39.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:38 smithi078 ceph-mon[21828]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:11:39.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:38 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:39.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:38 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:39.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:38 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:39.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:38 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-25T04:11:39.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:38 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:39.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:39 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-25T04:11:39.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:39 smithi078 ceph-mon[21828]: Deploying daemon grafana.smithi078 on smithi078 2024-04-25T04:11:39.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:39 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:41.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:40 smithi078 ceph-mon[21828]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:41.636 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2024-04-25T04:11:42.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:41 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1036003275' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-25T04:11:42.534 INFO:tasks.cephadm:Adding mon.smithi078 on smithi078 2024-04-25T04:11:42.534 INFO:tasks.cephadm:Adding mon.smithi112 on smithi112 2024-04-25T04:11:42.534 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch apply mon '2;smithi078:172.21.15.78=smithi078;smithi112:172.21.15.112=smithi112' 2024-04-25T04:11:42.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:42 smithi078 ceph-mon[21828]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:42.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:42 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1036003275' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-25T04:11:42.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:42 smithi078 ceph-mon[21828]: osdmap e4: 0 total, 0 up, 0 in 2024-04-25T04:11:44.211 INFO:teuthology.orchestra.run.smithi112.stdout:Scheduled mon update... 2024-04-25T04:11:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:44 smithi078 ceph-mon[21828]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:44.931 DEBUG:teuthology.orchestra.run.smithi112:mon.smithi112> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi112.service 2024-04-25T04:11:44.933 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:11:44.933 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:11:45.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:45 smithi078 ceph-mon[21828]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi078:172.21.15.78=smithi078;smithi112:172.21.15.112=smithi112", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:11:45.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:45 smithi078 ceph-mon[21828]: Saving service mon spec with placement smithi078:172.21.15.78=smithi078;smithi112:172.21.15.112=smithi112;count:2 2024-04-25T04:11:45.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:45 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:46.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:46 smithi078 ceph-mon[21828]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:47.098 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:11:47.098 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:11:47.100 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:11:47.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:47 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/114022804' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:11:48.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:48 smithi078 ceph-mon[21828]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:48.623 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:11:48.624 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:11:50.517 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:11:50.518 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:11:50.519 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:11:50.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:50 smithi078 ceph-mon[21828]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:51.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:51 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/2239851746' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:11:52.115 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:11:52.115 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:11:52.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:52 smithi078 ceph-mon[21828]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:53.843 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:11:53.843 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:11:53.844 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:11:54.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:54 smithi078 ceph-mon[21828]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:54.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:54 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/3143243965' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:11:55.440 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:11:55.440 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:11:56.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:56 smithi078 ceph-mon[21828]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:56.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:56 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:11:56.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:56 smithi078 ceph-mon[21828]: Deploying daemon node-exporter.smithi078 on smithi078 2024-04-25T04:11:57.198 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:11:57.198 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:11:57.200 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:11:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:57 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/4246466227' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:11:58.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:58 smithi078 ceph-mon[21828]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:11:58.827 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:11:58.827 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:00.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:59 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:00.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:59 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-25T04:12:00.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:11:59 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:00.559 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:00.559 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:00.560 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:01.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:00 smithi078 ceph-mon[21828]: from='mgr.14162 172.21.15.78:0/3230053378' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-25T04:12:01.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:00 smithi078 ceph-mon[21828]: mgrmap e14: smithi078.eugwik(active, since 50s) 2024-04-25T04:12:01.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:00 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/2111005955' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:02.138 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:02.138 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:04.010 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:04.010 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:04.012 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:04.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:04 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/3154440023' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:05.661 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:05.662 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:05.996 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:05 smithi078 ceph-mon[21828]: Active manager daemon smithi078.eugwik restarted 2024-04-25T04:12:05.996 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:05 smithi078 ceph-mon[21828]: Activating manager daemon smithi078.eugwik 2024-04-25T04:12:05.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:05 smithi078 ceph-mon[21828]: osdmap e5: 0 total, 0 up, 0 in 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: mgrmap e15: smithi078.eugwik(active, starting, since 0.0528282s) 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:12:06.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:12:06.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:06.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:12:06.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:06.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:12:06.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:12:06.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:06.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:07.606 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:07.607 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:07.608 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:08.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:07 smithi078 ceph-mon[21828]: [25/Apr/2024:04:12:06] ENGINE Bus STARTING 2024-04-25T04:12:08.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:07 smithi078 ceph-mon[21828]: [25/Apr/2024:04:12:06] ENGINE Serving on https://172.21.15.78:7150 2024-04-25T04:12:08.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:07 smithi078 ceph-mon[21828]: [25/Apr/2024:04:12:06] ENGINE Bus STARTED 2024-04-25T04:12:08.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:07 smithi078 ceph-mon[21828]: mgrmap e16: smithi078.eugwik(active, since 1.05829s) 2024-04-25T04:12:08.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:07 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:08.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:07 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/2686486107' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:09.211 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:09.212 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:09.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:08 smithi078 ceph-mon[21828]: mgrmap e17: smithi078.eugwik(active, since 2s) 2024-04-25T04:12:09.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:11.190 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:11.190 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:11.191 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:11.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:11 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/1775320484' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:12.867 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:12.867 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:13.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:13 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:13.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:13 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:13.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:13 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd/host:smithi112", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:12:14.575 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:14.575 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:14.577 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:12:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/1748304323' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:14.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:12:14.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-25T04:12:14.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:16.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:15 smithi078 ceph-mon[21828]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:12:16.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:15 smithi078 ceph-mon[21828]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:12:16.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:15 smithi078 ceph-mon[21828]: Deploying daemon crash.smithi112 on smithi112 2024-04-25T04:12:16.281 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:16.281 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:18.347 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:18.347 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:18.349 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:19.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:19 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/2042333946' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:19.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:19 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:19.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:19 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:12:19.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:19 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-25T04:12:19.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:19 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:12:19.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:19 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:19.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:19 smithi078 ceph-mon[21828]: Deploying daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:12:20.080 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:20.080 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:22.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:22.110 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:22.110 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:22.130 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:23.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:22 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/3331326825' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:23.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:23.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:12:23.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:23.906 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:23.906 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:24.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:23 smithi078 ceph-mon[21828]: Deploying daemon mon.smithi112 on smithi112 2024-04-25T04:12:26.878 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:26.879 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":1,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:10:09.587319Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-25T04:12:26.880 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 1 2024-04-25T04:12:27.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:26 smithi078 ceph-mon[21828]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:27.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:27.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:27 smithi112 ceph-mon[24160]: mon.smithi112@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-25T04:12:28.052 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:27 smithi112 bash[24120]: 179da40ce14ca95e5fa5450bf64a21a5e54158a3957e43b7450c0e45aa9eb5e1 2024-04-25T04:12:28.053 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:27 smithi112 systemd[1]: Started Ceph mon.smithi112 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:12:28.515 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-25T04:12:28.516 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-25T04:12:32.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:32.961 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: mon.smithi078 calling monitor election 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: mon.smithi112 calling monitor election 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:32.962 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: mon.smithi078 is new leader, mons smithi078,smithi112 in quorum (ranks 0,1) 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: monmap e2: 2 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],smithi112=[v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0]} 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: fsmap 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: osdmap e5: 0 total, 0 up, 0 in 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: mgrmap e17: smithi078.eugwik(active, since 26s) 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: overall HEALTH_OK 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:12:32.963 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: mon.smithi078 calling monitor election 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: mon.smithi112 calling monitor election 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: mon.smithi078 is new leader, mons smithi078,smithi112 in quorum (ranks 0,1) 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: monmap e2: 2 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],smithi112=[v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0]} 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: fsmap 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: osdmap e5: 0 total, 0 up, 0 in 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: mgrmap e17: smithi078.eugwik(active, since 26s) 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: overall HEALTH_OK 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:12:33.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/1009995110' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:12:33.107 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:33.436 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:12:33.436 INFO:teuthology.orchestra.run.smithi112.stdout:{"epoch":2,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","modified":"2024-04-25T04:12:27.641249Z","created":"2024-04-25T04:10:09.587319Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi112","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:3300","nonce":0},{"type":"v1","addr":"172.21.15.112:6789","nonce":0}]},"addr":"172.21.15.112:6789/0","public_addr":"172.21.15.112:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-25T04:12:33.437 INFO:teuthology.orchestra.run.smithi112.stderr:dumped monmap epoch 2 2024-04-25T04:12:33.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:33 smithi078 ceph-mon[21828]: Deploying daemon node-exporter.smithi112 on smithi112 2024-04-25T04:12:33.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:33 smithi078 ceph-mon[21828]: mgrmap e18: smithi078.eugwik(active, since 27s), standbys: smithi112.lvtlnw 2024-04-25T04:12:33.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:33 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:12:33.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:33 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/967920841' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:33.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:33 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:34.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:33 smithi112 ceph-mon[24160]: Deploying daemon node-exporter.smithi112 on smithi112 2024-04-25T04:12:34.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:33 smithi112 ceph-mon[24160]: mgrmap e18: smithi078.eugwik(active, since 27s), standbys: smithi112.lvtlnw 2024-04-25T04:12:34.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:33 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:12:34.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:33 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/967920841' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-25T04:12:34.013 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:33 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:12:34.036 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-25T04:12:34.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-25T04:12:35.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:34 smithi078 ceph-mon[21828]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:35.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:34 smithi112 ceph-mon[24160]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:35.920 INFO:teuthology.orchestra.run.smithi078.stdout:# minimal ceph.conf for 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:12:35.920 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-04-25T04:12:35.920 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:12:35.920 INFO:teuthology.orchestra.run.smithi078.stdout: mon_host = [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] 2024-04-25T04:12:36.511 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-25T04:12:36.511 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:12:36.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.conf 2024-04-25T04:12:36.536 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:12:36.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:12:36.600 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-25T04:12:36.601 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/ceph/ceph.conf 2024-04-25T04:12:36.632 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-25T04:12:36.633 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:12:36.699 INFO:tasks.cephadm:Deploying OSDs... 2024-04-25T04:12:36.699 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:12:36.699 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-04-25T04:12:36.714 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-25T04:12:36.714 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-04-25T04:12:36.769 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-25T04:12:36.769 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:36.770 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-25T04:12:36.770 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:36.770 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:36.770 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-04-25 04:11:56.067453543 +0000 2024-04-25T04:12:36.770 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-04-25 04:09:06.869726778 +0000 2024-04-25T04:12:36.770 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-04-25 04:09:06.869726778 +0000 2024-04-25T04:12:36.770 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-04-25 04:09:06.869726778 +0000 2024-04-25T04:12:36.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-25T04:12:36.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:36 smithi078 ceph-mon[21828]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:36.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:36 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:36.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:36 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/86744059' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:36.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:36 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:36.832 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-04-25T04:12:36.832 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-04-25T04:12:36.832 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000215287 s, 2.4 MB/s 2024-04-25T04:12:36.833 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-25T04:12:36.889 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-04-25T04:12:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:36 smithi112 ceph-mon[24160]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:36 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:36 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/86744059' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:36 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:36.944 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-25T04:12:36.944 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:36.944 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-25T04:12:36.945 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:36.945 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:36.945 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-04-25 04:11:56.067453543 +0000 2024-04-25T04:12:36.945 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-04-25 04:09:07.210721798 +0000 2024-04-25T04:12:36.945 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-04-25 04:09:07.210721798 +0000 2024-04-25T04:12:36.945 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-04-25 04:09:07.210721798 +0000 2024-04-25T04:12:36.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-25T04:12:37.007 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-04-25T04:12:37.007 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-04-25T04:12:37.007 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000186617 s, 2.7 MB/s 2024-04-25T04:12:37.008 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-25T04:12:37.063 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-04-25T04:12:37.118 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-04-25 04:11:56.067453543 +0000 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-04-25 04:09:07.550716833 +0000 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-04-25 04:09:07.550716833 +0000 2024-04-25T04:12:37.119 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-04-25 04:09:07.550716833 +0000 2024-04-25T04:12:37.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-25T04:12:37.180 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-04-25T04:12:37.181 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-04-25T04:12:37.181 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000179451 s, 2.9 MB/s 2024-04-25T04:12:37.182 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-25T04:12:37.237 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-04-25 04:11:56.068453529 +0000 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-04-25 04:09:07.890711868 +0000 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-04-25 04:09:07.890711868 +0000 2024-04-25T04:12:37.291 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-04-25 04:09:07.890711868 +0000 2024-04-25T04:12:37.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-25T04:12:37.352 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-04-25T04:12:37.352 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-04-25T04:12:37.352 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000178123 s, 2.9 MB/s 2024-04-25T04:12:37.353 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-25T04:12:37.410 DEBUG:teuthology.orchestra.run.smithi112:> set -ex 2024-04-25T04:12:37.410 DEBUG:teuthology.orchestra.run.smithi112:> dd if=/scratch_devs of=/dev/stdout 2024-04-25T04:12:37.429 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-25T04:12:37.429 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_1 2024-04-25T04:12:37.486 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-25T04:12:37.486 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:37.486 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-25T04:12:37.487 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:37.487 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:37.487 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-25 04:12:33.242745201 +0000 2024-04-25T04:12:37.487 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-25 04:09:06.867031412 +0000 2024-04-25T04:12:37.487 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-25 04:09:06.867031412 +0000 2024-04-25T04:12:37.487 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-25 04:09:06.867031412 +0000 2024-04-25T04:12:37.487 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-25T04:12:37.553 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-25T04:12:37.554 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-25T04:12:37.554 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000290563 s, 1.8 MB/s 2024-04-25T04:12:37.555 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-25T04:12:37.613 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_2 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-25 04:12:33.242745201 +0000 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-25 04:09:07.209022344 +0000 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-25 04:09:07.209022344 +0000 2024-04-25T04:12:37.670 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-25 04:09:07.209022344 +0000 2024-04-25T04:12:37.671 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-25T04:12:37.735 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-25T04:12:37.736 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-25T04:12:37.736 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000215285 s, 2.4 MB/s 2024-04-25T04:12:37.737 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-25T04:12:37.795 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_3 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-25 04:12:33.242745201 +0000 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-25 04:09:07.555013170 +0000 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-25 04:09:07.555013170 +0000 2024-04-25T04:12:37.851 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-25 04:09:07.555013170 +0000 2024-04-25T04:12:37.852 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-25T04:12:37.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:37 smithi112 ceph-mon[24160]: Deploying daemon prometheus.smithi078 on smithi078 2024-04-25T04:12:37.914 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-25T04:12:37.915 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-25T04:12:37.915 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000204801 s, 2.5 MB/s 2024-04-25T04:12:37.916 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-25T04:12:37.973 DEBUG:teuthology.orchestra.run.smithi112:> stat /dev/vg_nvme/lv_4 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout:Context: system_u:object_r:device_t:s0 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout:Access: 2024-04-25 04:12:33.243745174 +0000 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout:Modify: 2024-04-25 04:09:07.890004287 +0000 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout:Change: 2024-04-25 04:09:07.890004287 +0000 2024-04-25T04:12:38.029 INFO:teuthology.orchestra.run.smithi112.stdout: Birth: 2024-04-25 04:09:07.890004287 +0000 2024-04-25T04:12:38.029 DEBUG:teuthology.orchestra.run.smithi112:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-25T04:12:38.092 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records in 2024-04-25T04:12:38.092 INFO:teuthology.orchestra.run.smithi112.stderr:1+0 records out 2024-04-25T04:12:38.092 INFO:teuthology.orchestra.run.smithi112.stderr:512 bytes copied, 0.000256085 s, 2.0 MB/s 2024-04-25T04:12:38.094 DEBUG:teuthology.orchestra.run.smithi112:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-25T04:12:38.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:37 smithi078 ceph-mon[21828]: Deploying daemon prometheus.smithi078 on smithi078 2024-04-25T04:12:38.150 INFO:tasks.cephadm:Deploying osd.0 on smithi078 with /dev/vg_nvme/lv_4... 2024-04-25T04:12:38.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-04-25T04:12:39.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:38 smithi078 ceph-mon[21828]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:38 smithi112 ceph-mon[24160]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:41.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:40 smithi078 ceph-mon[21828]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:40 smithi112 ceph-mon[24160]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:42.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:41 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:42.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:41 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:43.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:42 smithi078 ceph-mon[21828]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:42 smithi112 ceph-mon[24160]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:44 smithi078 ceph-mon[21828]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:44 smithi112 ceph-mon[24160]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:47.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:46 smithi078 ceph-mon[21828]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:47.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:46 smithi112 ceph-mon[24160]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:49.055 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:48 smithi078 ceph-mon[21828]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:49.091 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:48 smithi112 ceph-mon[24160]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:50.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:50.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:12:50.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:50.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:12:50.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:50 smithi078 ceph-mon[21828]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:50.328 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:50.328 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:12:50.328 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:50.329 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:12:50.329 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:50 smithi112 ceph-mon[24160]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:50.963 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:12:50.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_4 2024-04-25T04:12:51.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:51.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:51 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:12:51.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:51.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:52.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:52.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:51 smithi078 ceph-mon[21828]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:12:52.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:52.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:53.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:52 smithi078 ceph-mon[21828]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:12:53.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:52 smithi078 ceph-mon[21828]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:52 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:12:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:52 smithi112 ceph-mon[24160]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:54.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:54 smithi078 ceph-mon[21828]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:54.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:54 smithi112 ceph-mon[24160]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:55.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:55 smithi078 ceph-mon[21828]: from='client.14240 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:12:55.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:55 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:12:55.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:55 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:12:55.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:55 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:55.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:55 smithi112 ceph-mon[24160]: from='client.14240 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:12:55.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:55 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:12:55.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:55 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:12:55.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:55 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: Reconfiguring daemon mon.smithi078 on smithi078 2024-04-25T04:12:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:56 smithi078 ceph-mon[21828]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: Reconfiguring daemon mon.smithi078 on smithi078 2024-04-25T04:12:56.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:56 smithi112 ceph-mon[24160]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:58.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:58.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: Reconfiguring mgr.smithi078.eugwik (unknown last config time)... 2024-04-25T04:12:58.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:12:58.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:12:58.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:58.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: Reconfiguring daemon mgr.smithi078.eugwik on smithi078 2024-04-25T04:12:58.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:58.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/981239629' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d61a94e-0810-4c64-a942-e6dad01543d8"}]: dispatch 2024-04-25T04:12:58.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/981239629' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d61a94e-0810-4c64-a942-e6dad01543d8"}]': finished 2024-04-25T04:12:58.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: osdmap e6: 1 total, 0 up, 1 in 2024-04-25T04:12:58.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:12:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:12:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: Reconfiguring mgr.smithi078.eugwik (unknown last config time)... 2024-04-25T04:12:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:12:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:12:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:12:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: Reconfiguring daemon mgr.smithi078.eugwik on smithi078 2024-04-25T04:12:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:12:58.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/981239629' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d61a94e-0810-4c64-a942-e6dad01543d8"}]: dispatch 2024-04-25T04:12:58.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/981239629' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d61a94e-0810-4c64-a942-e6dad01543d8"}]': finished 2024-04-25T04:12:58.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: osdmap e6: 1 total, 0 up, 1 in 2024-04-25T04:12:58.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:12:59.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:12:59 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2994521893' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:12:59.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:12:59 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2994521893' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:13:00.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:00 smithi078 ceph-mon[21828]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:00.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:00 smithi112 ceph-mon[24160]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:03.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:02 smithi078 ceph-mon[21828]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:02 smithi112 ceph-mon[24160]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:04.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:04 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:04.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:04 smithi078 ceph-mon[21828]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-04-25T04:13:04.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:04 smithi078 ceph-mon[21828]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-04-25T04:13:04.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:04 smithi078 ceph-mon[21828]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:04 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:04 smithi112 ceph-mon[24160]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-04-25T04:13:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:04 smithi112 ceph-mon[24160]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-04-25T04:13:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:04 smithi112 ceph-mon[24160]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:06.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:13:06.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:13:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:13:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:13:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:06 smithi078 ceph-mon[21828]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:06 smithi112 ceph-mon[24160]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:09.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:08 smithi078 ceph-mon[21828]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:09.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:09.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:13:09.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:09.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:08 smithi112 ceph-mon[24160]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:09.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:09.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:13:09.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:10.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:09 smithi078 ceph-mon[21828]: Reconfiguring crash.smithi078 (monmap changed)... 2024-04-25T04:13:10.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:09 smithi078 ceph-mon[21828]: Reconfiguring daemon crash.smithi078 on smithi078 2024-04-25T04:13:10.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:09 smithi112 ceph-mon[24160]: Reconfiguring crash.smithi078 (monmap changed)... 2024-04-25T04:13:10.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:09 smithi112 ceph-mon[24160]: Reconfiguring daemon crash.smithi078 on smithi078 2024-04-25T04:13:10.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:10 smithi078 ceph-mon[21828]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:10.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T04:13:10.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:11.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:10 smithi112 ceph-mon[24160]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:11.161 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T04:13:11.161 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:12.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:11 smithi078 ceph-mon[21828]: Deploying daemon osd.0 on smithi078 2024-04-25T04:13:12.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:11 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:12.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:11 smithi112 ceph-mon[24160]: Deploying daemon osd.0 on smithi078 2024-04-25T04:13:12.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:11 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:13.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:12 smithi078 ceph-mon[21828]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-04-25T04:13:13.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:12 smithi078 ceph-mon[21828]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-04-25T04:13:13.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:12 smithi078 ceph-mon[21828]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:12 smithi112 ceph-mon[24160]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-04-25T04:13:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:12 smithi112 ceph-mon[24160]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-04-25T04:13:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:12 smithi112 ceph-mon[24160]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:14.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:14 smithi112 ceph-mon[24160]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:14.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:14 smithi078 ceph-mon[21828]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:16.952 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:16 smithi078 ceph-mon[21828]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:16.952 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:16 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:16 smithi112 ceph-mon[24160]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:16 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:19.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:18 smithi078 ceph-mon[21828]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:18 smithi112 ceph-mon[24160]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:19 smithi078 ceph-mon[21828]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-25T04:13:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:19 smithi112 ceph-mon[24160]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-25T04:13:21.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:20 smithi078 ceph-mon[21828]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:21.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:20 smithi078 ceph-mon[21828]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-25T04:13:21.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:20 smithi078 ceph-mon[21828]: osdmap e7: 1 total, 0 up, 1 in 2024-04-25T04:13:21.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:20 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:21.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:20 smithi078 ceph-mon[21828]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:13:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:20 smithi112 ceph-mon[24160]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:20 smithi112 ceph-mon[24160]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-25T04:13:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:20 smithi112 ceph-mon[24160]: osdmap e7: 1 total, 0 up, 1 in 2024-04-25T04:13:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:20 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:20 smithi112 ceph-mon[24160]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:13:21.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:21 smithi112 ceph-mon[24160]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:13:21.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:21 smithi112 ceph-mon[24160]: osdmap e8: 1 total, 0 up, 1 in 2024-04-25T04:13:21.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:21 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:21.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:21 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:21.891 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:21 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:21.891 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:21 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:13:21.891 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:21 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:22.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:21 smithi078 ceph-mon[21828]: from='osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:13:22.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:21 smithi078 ceph-mon[21828]: osdmap e8: 1 total, 0 up, 1 in 2024-04-25T04:13:22.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:22.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:22.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:22.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:13:22.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:22.774 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 0 on host 'smithi078' 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107] boot 2024-04-25T04:13:23.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: osdmap e9: 1 total, 1 up, 1 in 2024-04-25T04:13:23.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:23.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:23.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:13:23.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:13:23.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:23.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:23.070 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:13:23.070 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:13:23.070 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-25T04:13:23.070 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-25T04:13:23.070 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: osd.0 [v2:172.21.15.78:6802/2365438107,v1:172.21.15.78:6803/2365438107] boot 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: osdmap e9: 1 total, 1 up, 1 in 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:23.071 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:23.363 DEBUG:teuthology.orchestra.run.smithi078:osd.0> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.0.service 2024-04-25T04:13:23.367 INFO:tasks.cephadm:Deploying osd.1 on smithi078 with /dev/vg_nvme/lv_3... 2024-04-25T04:13:23.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-04-25T04:13:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:23 smithi112 ceph-mon[24160]: Reconfiguring mgr.smithi112.lvtlnw (monmap changed)... 2024-04-25T04:13:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:23 smithi112 ceph-mon[24160]: Reconfiguring daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:13:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:23 smithi112 ceph-mon[24160]: osdmap e10: 1 total, 1 up, 1 in 2024-04-25T04:13:23.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:23 smithi078 ceph-mon[21828]: Reconfiguring mgr.smithi112.lvtlnw (monmap changed)... 2024-04-25T04:13:23.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:23 smithi078 ceph-mon[21828]: Reconfiguring daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:13:23.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:23 smithi078 ceph-mon[21828]: osdmap e10: 1 total, 1 up, 1 in 2024-04-25T04:13:24.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:24 smithi078 ceph-mon[21828]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:24.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:24.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:24 smithi078 ceph-mon[21828]: Reconfiguring mon.smithi112 (monmap changed)... 2024-04-25T04:13:24.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:13:24.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:13:24.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:24.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:24 smithi078 ceph-mon[21828]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-25T04:13:25.048 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:24 smithi112 ceph-mon[24160]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:25.048 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:25.048 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:24 smithi112 ceph-mon[24160]: Reconfiguring mon.smithi112 (monmap changed)... 2024-04-25T04:13:25.048 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:13:25.048 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:13:25.048 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:25.048 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:24 smithi112 ceph-mon[24160]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-04-25T04:13:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:13:26.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:26 smithi078 ceph-mon[21828]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:26.384 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-04-25T04:13:26.385 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-04-25T04:13:26.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:26.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:13:26.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:26.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:13:26.386 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:26 smithi112 ceph-mon[24160]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:26.696 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:13:26.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_3 2024-04-25T04:13:27.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:27 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:28.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:27 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:29.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:28 smithi078 ceph-mon[21828]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:29.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:28 smithi112 ceph-mon[24160]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:30.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:29 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:13:30.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:29 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:13:30.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:29 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:30.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:29 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:13:30.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:29 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:13:30.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:29 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:31.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:30 smithi078 ceph-mon[21828]: from='client.14258 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:13:31.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:30 smithi078 ceph-mon[21828]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:30 smithi112 ceph-mon[24160]: from='client.14258 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:13:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:30 smithi112 ceph-mon[24160]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:33.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:32 smithi078 ceph-mon[21828]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:33.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:32 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/4187797574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a27d6b5e-7d58-49fd-9f55-896f3d37f68d"}]: dispatch 2024-04-25T04:13:33.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:32 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/4187797574' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a27d6b5e-7d58-49fd-9f55-896f3d37f68d"}]': finished 2024-04-25T04:13:33.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:32 smithi078 ceph-mon[21828]: osdmap e11: 2 total, 1 up, 2 in 2024-04-25T04:13:33.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:33.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:32 smithi112 ceph-mon[24160]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:33.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:32 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/4187797574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a27d6b5e-7d58-49fd-9f55-896f3d37f68d"}]: dispatch 2024-04-25T04:13:33.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:32 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/4187797574' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a27d6b5e-7d58-49fd-9f55-896f3d37f68d"}]': finished 2024-04-25T04:13:33.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:32 smithi112 ceph-mon[24160]: osdmap e11: 2 total, 1 up, 2 in 2024-04-25T04:13:33.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:34.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:33 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:34.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:33 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1624253920' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:13:34.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:33 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:34.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:33 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/1624253920' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:13:35.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:34 smithi078 ceph-mon[21828]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:35.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:34 smithi112 ceph-mon[24160]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:37.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:36 smithi078 ceph-mon[21828]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:37.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:36 smithi112 ceph-mon[24160]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:38.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:38.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:38.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:13:38.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:38.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:38.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:13:38.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:38.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:13:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:13:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:13:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:39.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:13:40.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:39 smithi078 ceph-mon[21828]: Detected new or changed devices on smithi078 2024-04-25T04:13:40.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:39 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi078 to 12864M 2024-04-25T04:13:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:39 smithi112 ceph-mon[24160]: Detected new or changed devices on smithi078 2024-04-25T04:13:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:39 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi078 to 12864M 2024-04-25T04:13:40.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:40 smithi078 ceph-mon[21828]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:40.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:40 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:40.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:40 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T04:13:40.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:40 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:40 smithi112 ceph-mon[24160]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:40 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:40 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T04:13:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:40 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:42.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:41 smithi078 ceph-mon[21828]: Deploying daemon osd.1 on smithi078 2024-04-25T04:13:42.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:41 smithi112 ceph-mon[24160]: Deploying daemon osd.1 on smithi078 2024-04-25T04:13:42.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:42 smithi078 ceph-mon[21828]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:42 smithi112 ceph-mon[24160]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:44 smithi112 ceph-mon[24160]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:44.410 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:44 smithi078 ceph-mon[21828]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:46.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:46 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:46.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:46 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:46.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:46 smithi078 ceph-mon[21828]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:46 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:46 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:46 smithi112 ceph-mon[24160]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:48.886 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:48 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:48.886 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:48 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:13:48.886 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:48 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:48.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:48 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:13:48.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:48 smithi112 ceph-mon[24160]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:49.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:48 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:49.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:48 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:13:49.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:48 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:49.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:48 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:13:49.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:48 smithi078 ceph-mon[21828]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:50.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:50.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:50 smithi078 ceph-mon[21828]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:50.507 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 1 on host 'smithi078' 2024-04-25T04:13:50.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:50.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:50 smithi112 ceph-mon[24160]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:51.159 DEBUG:teuthology.orchestra.run.smithi078:osd.1> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.1.service 2024-04-25T04:13:51.162 INFO:tasks.cephadm:Deploying osd.2 on smithi078 with /dev/vg_nvme/lv_2... 2024-04-25T04:13:51.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-04-25T04:13:51.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:51.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:51 smithi078 ceph-mon[21828]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T04:13:51.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:13:51.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:51 smithi112 ceph-mon[24160]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T04:13:52.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-mon[21828]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-25T04:13:52.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-mon[21828]: osdmap e12: 2 total, 1 up, 2 in 2024-04-25T04:13:52.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:52.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-mon[21828]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:13:52.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-mon[21828]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:52.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-mon[21828]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:13:52.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-mon[21828]: osdmap e13: 2 total, 1 up, 2 in 2024-04-25T04:13:52.587 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[36314]: 2024-04-25T04:13:52.507+0000 7f1eb0539700 -1 osd.1 0 waiting for initial osdmap 2024-04-25T04:13:52.587 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:13:52 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[36314]: 2024-04-25T04:13:52.510+0000 7f1eabed2700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:13:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:52 smithi112 ceph-mon[24160]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-25T04:13:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:52 smithi112 ceph-mon[24160]: osdmap e12: 2 total, 1 up, 2 in 2024-04-25T04:13:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:52 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:52 smithi112 ceph-mon[24160]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:13:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:52 smithi112 ceph-mon[24160]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:52 smithi112 ceph-mon[24160]: from='osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:13:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:52 smithi112 ceph-mon[24160]: osdmap e13: 2 total, 1 up, 2 in 2024-04-25T04:13:53.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:53.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:53.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:53.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:53.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:53.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:54.037 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:13:54.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_2 2024-04-25T04:13:54.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:54 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:13:54.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:54 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:13:54.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:54 smithi078 ceph-mon[21828]: osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174] boot 2024-04-25T04:13:54.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:54 smithi078 ceph-mon[21828]: osdmap e14: 2 total, 2 up, 2 in 2024-04-25T04:13:54.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:54 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:54.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:54 smithi078 ceph-mon[21828]: pgmap v56: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:54 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:13:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:54 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:13:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:54 smithi112 ceph-mon[24160]: osd.1 [v2:172.21.15.78:6810/334381174,v1:172.21.15.78:6811/334381174] boot 2024-04-25T04:13:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:54 smithi112 ceph-mon[24160]: osdmap e14: 2 total, 2 up, 2 in 2024-04-25T04:13:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:54 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:13:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:54 smithi112 ceph-mon[24160]: pgmap v56: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-25T04:13:55.683 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:55 smithi078 ceph-mon[21828]: osdmap e15: 2 total, 2 up, 2 in 2024-04-25T04:13:55.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:55 smithi112 ceph-mon[24160]: osdmap e15: 2 total, 2 up, 2 in 2024-04-25T04:13:56.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:56 smithi078 ceph-mon[21828]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:13:56.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:13:56.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:13:56.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:56 smithi112 ceph-mon[24160]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:13:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:13:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:13:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:13:57.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:57 smithi078 ceph-mon[21828]: from='client.14276 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:13:57.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:57 smithi112 ceph-mon[24160]: from='client.14276 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:13:58.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:13:58 smithi078 ceph-mon[21828]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:13:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:13:58 smithi112 ceph-mon[24160]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:00.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:00 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/3348543402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3231645-dee2-4b48-bb4a-28000e933441"}]: dispatch 2024-04-25T04:14:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:00 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/3348543402' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3231645-dee2-4b48-bb4a-28000e933441"}]': finished 2024-04-25T04:14:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:00 smithi112 ceph-mon[24160]: osdmap e16: 3 total, 2 up, 3 in 2024-04-25T04:14:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:00 smithi112 ceph-mon[24160]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:00 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/49990773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:14:00.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:00.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:00 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3348543402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3231645-dee2-4b48-bb4a-28000e933441"}]: dispatch 2024-04-25T04:14:00.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:00 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3348543402' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3231645-dee2-4b48-bb4a-28000e933441"}]': finished 2024-04-25T04:14:00.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:00 smithi078 ceph-mon[21828]: osdmap e16: 3 total, 2 up, 3 in 2024-04-25T04:14:00.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:00.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:00 smithi078 ceph-mon[21828]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:00.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:00 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/49990773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:14:03.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:02 smithi078 ceph-mon[21828]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:02 smithi112 ceph-mon[24160]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:04.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:04 smithi078 ceph-mon[21828]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:04.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:04 smithi112 ceph-mon[24160]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:06.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: Detected new or changed devices on smithi078 2024-04-25T04:14:06.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:06.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:06.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:06.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi078 to 6432M 2024-04-25T04:14:06.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:06.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:06.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:06.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:06.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:06.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:06.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:14:06.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:14:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: Detected new or changed devices on smithi078 2024-04-25T04:14:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi078 to 6432M 2024-04-25T04:14:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:14:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:14:07.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:07 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T04:14:07.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:07 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:07.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:07 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T04:14:07.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:07 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:08.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:08 smithi078 ceph-mon[21828]: Deploying daemon osd.2 on smithi078 2024-04-25T04:14:08.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:08 smithi078 ceph-mon[21828]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:08.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:08 smithi112 ceph-mon[24160]: Deploying daemon osd.2 on smithi078 2024-04-25T04:14:08.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:08 smithi112 ceph-mon[24160]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:11.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:10 smithi078 ceph-mon[21828]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:11.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:10 smithi112 ceph-mon[24160]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:13.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:12 smithi078 ceph-mon[21828]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:13.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:13.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:12 smithi112 ceph-mon[24160]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:14.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:14 smithi078 ceph-mon[21828]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:14.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:14 smithi112 ceph-mon[24160]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:16.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:15 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:16.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:15 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:16.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:15 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:16.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:15 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:16.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:15 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:16.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:15 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:16.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:15 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:16.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:15 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:16.982 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:16 smithi078 ceph-mon[21828]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:16 smithi112 ceph-mon[24160]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 2 on host 'smithi078' 2024-04-25T04:14:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:18 smithi078 ceph-mon[21828]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:18.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:18 smithi078 ceph-mon[21828]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T04:14:18.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:18 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:18.925 DEBUG:teuthology.orchestra.run.smithi078:osd.2> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.2.service 2024-04-25T04:14:18.927 INFO:tasks.cephadm:Deploying osd.3 on smithi078 with /dev/vg_nvme/lv_1... 2024-04-25T04:14:18.927 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-04-25T04:14:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:18 smithi112 ceph-mon[24160]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:18 smithi112 ceph-mon[24160]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T04:14:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:18 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:19 smithi078 ceph-mon[21828]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-25T04:14:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:19 smithi078 ceph-mon[21828]: osdmap e17: 3 total, 2 up, 3 in 2024-04-25T04:14:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:19 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:19 smithi078 ceph-mon[21828]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:14:20.105 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[40010]: 2024-04-25T04:14:19.806+0000 7f03f0bf8700 -1 osd.2 0 waiting for initial osdmap 2024-04-25T04:14:20.105 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[40010]: 2024-04-25T04:14:19.812+0000 7f03ec591700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:14:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:19 smithi112 ceph-mon[24160]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-25T04:14:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:19 smithi112 ceph-mon[24160]: osdmap e17: 3 total, 2 up, 3 in 2024-04-25T04:14:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:19 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:19 smithi112 ceph-mon[24160]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:14:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:20 smithi078 ceph-mon[21828]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:20 smithi078 ceph-mon[21828]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:14:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:20 smithi078 ceph-mon[21828]: osdmap e18: 3 total, 2 up, 3 in 2024-04-25T04:14:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:20 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:20 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:20 smithi112 ceph-mon[24160]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:20 smithi112 ceph-mon[24160]: from='osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:14:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:20 smithi112 ceph-mon[24160]: osdmap e18: 3 total, 2 up, 3 in 2024-04-25T04:14:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:20 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:20 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:22.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:22 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:14:22.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:22 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:14:22.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:22.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:22 smithi078 ceph-mon[21828]: osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465] boot 2024-04-25T04:14:22.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:22 smithi078 ceph-mon[21828]: osdmap e19: 3 total, 3 up, 3 in 2024-04-25T04:14:22.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:22.161 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:14:22.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_1 2024-04-25T04:14:22.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:22 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:14:22.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:22 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:14:22.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:22.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:22 smithi112 ceph-mon[24160]: osd.2 [v2:172.21.15.78:6818/3901142465,v1:172.21.15.78:6819/3901142465] boot 2024-04-25T04:14:22.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:22 smithi112 ceph-mon[24160]: osdmap e19: 3 total, 3 up, 3 in 2024-04-25T04:14:22.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:14:23.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:23 smithi078 ceph-mon[21828]: pgmap v75: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:23.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-25T04:14:23.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-25T04:14:23.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:23 smithi078 ceph-mon[21828]: osdmap e20: 3 total, 3 up, 3 in 2024-04-25T04:14:23.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T04:14:23.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:23 smithi112 ceph-mon[24160]: pgmap v75: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-25T04:14:23.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-25T04:14:23.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-25T04:14:23.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:23 smithi112 ceph-mon[24160]: osdmap e20: 3 total, 3 up, 3 in 2024-04-25T04:14:23.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-25T04:14:24.017 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:14:23 smithi078 sudo[41316]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T04:14:24.018 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:14:23 smithi078 sudo[41316]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:24.018 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:14:23 smithi078 sudo[41316]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:24.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-25T04:14:24.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:24 smithi078 ceph-mon[21828]: osdmap e21: 3 total, 3 up, 3 in 2024-04-25T04:14:24.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:24 smithi078 ceph-mon[21828]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:24.308 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41353]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T04:14:24.308 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41353]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:24.308 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41353]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-25T04:14:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:24 smithi112 ceph-mon[24160]: osdmap e21: 3 total, 3 up, 3 in 2024-04-25T04:14:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:24 smithi112 ceph-mon[24160]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:24.604 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41360]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T04:14:24.604 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41360]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:24.605 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41360]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:24.973 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41391]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T04:14:24.973 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41391]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:24.973 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41391]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:25.239 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41413]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T04:14:25.239 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41413]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:25.239 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:24 smithi078 sudo[41413]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:25.239 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41473]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T04:14:25.239 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41473]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:25.239 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 ceph-mon[21828]: mgrmap e19: smithi078.eugwik(active, since 2m), standbys: smithi112.lvtlnw 2024-04-25T04:14:25.240 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 ceph-mon[21828]: osdmap e22: 3 total, 3 up, 3 in 2024-04-25T04:14:25.240 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:25.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:25 smithi112 ceph-mon[24160]: mgrmap e19: smithi078.eugwik(active, since 2m), standbys: smithi112.lvtlnw 2024-04-25T04:14:25.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:25 smithi112 ceph-mon[24160]: osdmap e22: 3 total, 3 up, 3 in 2024-04-25T04:14:25.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:25 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:25.498 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41473]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:25.498 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41479]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T04:14:25.498 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41479]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:25.499 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41479]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:25.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41482]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T04:14:25.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41482]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:25.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:25 smithi078 sudo[41482]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:26.238 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:25 smithi112 sudo[26467]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-25T04:14:26.238 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:25 smithi112 sudo[26467]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:26.238 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:25 smithi112 sudo[26467]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:14:26.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:26 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 sudo[26470]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 sudo[26470]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 sudo[26470]: pam_unix(sudo:session): session closed for user root 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:14:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:14:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:14:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:14:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:14:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:14:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:26 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:27.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:27 smithi078 ceph-mon[21828]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T04:14:27.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:27 smithi078 ceph-mon[21828]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T04:14:27.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:27 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:27 smithi112 ceph-mon[24160]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-25T04:14:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:27 smithi112 ceph-mon[24160]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-25T04:14:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:27 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:28.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:28 smithi078 ceph-mon[21828]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:28.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:28 smithi112 ceph-mon[24160]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:31.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:31.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:31.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-25T04:14:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:32.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:31 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi078 to 4288M 2024-04-25T04:14:32.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:31 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/654259790' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3cc63613-0ceb-4585-aaf8-03283211d41c"}]: dispatch 2024-04-25T04:14:32.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:31 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/654259790' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3cc63613-0ceb-4585-aaf8-03283211d41c"}]': finished 2024-04-25T04:14:32.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:31 smithi078 ceph-mon[21828]: osdmap e23: 4 total, 3 up, 4 in 2024-04-25T04:14:32.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:31 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:32.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:31 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/341932232' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:14:32.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:31 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi078 to 4288M 2024-04-25T04:14:32.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:31 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/654259790' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3cc63613-0ceb-4585-aaf8-03283211d41c"}]: dispatch 2024-04-25T04:14:32.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:31 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/654259790' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3cc63613-0ceb-4585-aaf8-03283211d41c"}]': finished 2024-04-25T04:14:32.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:31 smithi112 ceph-mon[24160]: osdmap e23: 4 total, 3 up, 4 in 2024-04-25T04:14:32.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:31 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:32.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:31 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/341932232' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:14:33.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:32 smithi078 ceph-mon[21828]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-25T04:14:33.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:32 smithi112 ceph-mon[24160]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-25T04:14:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:34 smithi112 ceph-mon[24160]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-25T04:14:34.448 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:34 smithi078 ceph-mon[21828]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-25T04:14:37.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:36 smithi078 ceph-mon[21828]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-25T04:14:37.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:36 smithi112 ceph-mon[24160]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-25T04:14:38.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:37 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T04:14:38.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:37 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:38.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:37 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T04:14:38.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:37 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:38.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:38 smithi078 ceph-mon[21828]: Deploying daemon osd.3 on smithi078 2024-04-25T04:14:38.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:38 smithi078 ceph-mon[21828]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-25T04:14:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:38 smithi112 ceph-mon[24160]: Deploying daemon osd.3 on smithi078 2024-04-25T04:14:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:38 smithi112 ceph-mon[24160]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-25T04:14:41.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:40 smithi078 ceph-mon[21828]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:40 smithi112 ceph-mon[24160]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:43.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:42 smithi078 ceph-mon[21828]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:43.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:43.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:43.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:43.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:42 smithi112 ceph-mon[24160]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:44.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:44 smithi078 ceph-mon[21828]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:44 smithi112 ceph-mon[24160]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:45.637 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 3 on host 'smithi078' 2024-04-25T04:14:46.582 DEBUG:teuthology.orchestra.run.smithi078:osd.3> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.3.service 2024-04-25T04:14:46.584 INFO:tasks.cephadm:Deploying osd.4 on smithi112 with /dev/vg_nvme/lv_4... 2024-04-25T04:14:46.584 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-04-25T04:14:46.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:46 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:46.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:46 smithi078 ceph-mon[21828]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:46.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:46 smithi078 ceph-mon[21828]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-25T04:14:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:46 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:46 smithi112 ceph-mon[24160]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:46 smithi112 ceph-mon[24160]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-25T04:14:47.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:47 smithi078 ceph-mon[21828]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-25T04:14:47.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:47 smithi078 ceph-mon[21828]: osdmap e24: 4 total, 3 up, 4 in 2024-04-25T04:14:47.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:47 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:47.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:47 smithi078 ceph-mon[21828]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:14:47.878 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:14:47 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[43279]: 2024-04-25T04:14:47.645+0000 7f013ce3d700 -1 osd.3 0 waiting for initial osdmap 2024-04-25T04:14:47.878 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:14:47 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[43279]: 2024-04-25T04:14:47.651+0000 7f01397d8700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:14:47.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:47 smithi112 ceph-mon[24160]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-25T04:14:47.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:47 smithi112 ceph-mon[24160]: osdmap e24: 4 total, 3 up, 4 in 2024-04-25T04:14:47.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:47 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:47.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:47 smithi112 ceph-mon[24160]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:14:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:48 smithi112 ceph-mon[24160]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:14:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:48 smithi112 ceph-mon[24160]: osdmap e25: 4 total, 3 up, 4 in 2024-04-25T04:14:48.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:48 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:48.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:48 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:48.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:48 smithi112 ceph-mon[24160]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:48.949 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:14:48.969 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_4 2024-04-25T04:14:49.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:48 smithi078 ceph-mon[21828]: from='osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-04-25T04:14:49.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:48 smithi078 ceph-mon[21828]: osdmap e25: 4 total, 3 up, 4 in 2024-04-25T04:14:49.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:48 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:49.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:48 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:49.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:48 smithi078 ceph-mon[21828]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-25T04:14:49.782 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:49 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:14:49.783 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:49 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:14:49.783 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:49 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:49.783 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:49 smithi112 ceph-mon[24160]: osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235] boot 2024-04-25T04:14:49.783 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:49 smithi112 ceph-mon[24160]: osdmap e26: 4 total, 4 up, 4 in 2024-04-25T04:14:49.783 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:49 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:49.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:49 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:14:49.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:49 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:14:49.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:49 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:49.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:49 smithi078 ceph-mon[21828]: osd.3 [v2:172.21.15.78:6826/2527808235,v1:172.21.15.78:6827/2527808235] boot 2024-04-25T04:14:49.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:49 smithi078 ceph-mon[21828]: osdmap e26: 4 total, 4 up, 4 in 2024-04-25T04:14:49.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:49 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:14:51.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:50 smithi078 ceph-mon[21828]: osdmap e27: 4 total, 4 up, 4 in 2024-04-25T04:14:51.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:51.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:50 smithi078 ceph-mon[21828]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:51.137 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:50 smithi112 ceph-mon[24160]: osdmap e27: 4 total, 4 up, 4 in 2024-04-25T04:14:51.138 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:51.138 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:50 smithi112 ceph-mon[24160]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:51.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:51 smithi078 ceph-mon[21828]: from='client.24133 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:14:51.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:14:51.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:14:51.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:51.937 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:51 smithi112 ceph-mon[24160]: from='client.24133 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:14:51.937 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:14:51.937 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:14:51.937 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:52.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:52 smithi078 ceph-mon[21828]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:52 smithi112 ceph-mon[24160]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/415617153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb4c070a-b7ec-4753-b794-847844fa7604"}]: dispatch 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb4c070a-b7ec-4753-b794-847844fa7604"}]: dispatch 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb4c070a-b7ec-4753-b794-847844fa7604"}]': finished 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: osdmap e28: 5 total, 4 up, 5 in 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:54.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:54.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:54.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:53 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/393308712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:14:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/415617153' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb4c070a-b7ec-4753-b794-847844fa7604"}]: dispatch 2024-04-25T04:14:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb4c070a-b7ec-4753-b794-847844fa7604"}]: dispatch 2024-04-25T04:14:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb4c070a-b7ec-4753-b794-847844fa7604"}]': finished 2024-04-25T04:14:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: osdmap e28: 5 total, 4 up, 5 in 2024-04-25T04:14:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:14:54.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:53 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/393308712' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:14:55.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:54 smithi078 ceph-mon[21828]: Detected new or changed devices on smithi078 2024-04-25T04:14:55.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:54 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi078 to 3216M 2024-04-25T04:14:55.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:54 smithi078 ceph-mon[21828]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:55.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:54 smithi112 ceph-mon[24160]: Detected new or changed devices on smithi078 2024-04-25T04:14:55.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:54 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi078 to 3216M 2024-04-25T04:14:55.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:54 smithi112 ceph-mon[24160]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:57.025 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:56 smithi112 ceph-mon[24160]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:57.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:56 smithi078 ceph-mon[21828]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:58.868 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:58 smithi112 ceph-mon[24160]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:14:59.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:58 smithi078 ceph-mon[21828]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:00.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:59 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T04:15:00.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:59 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:00.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:14:59 smithi078 ceph-mon[21828]: Deploying daemon osd.4 on smithi112 2024-04-25T04:15:00.145 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:59 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T04:15:00.145 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:59 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:00.145 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:14:59 smithi112 ceph-mon[24160]: Deploying daemon osd.4 on smithi112 2024-04-25T04:15:01.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:00 smithi078 ceph-mon[21828]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:01.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:00 smithi112 ceph-mon[24160]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:03.042 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:02 smithi112 ceph-mon[24160]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:03.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:02 smithi078 ceph-mon[21828]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:04.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:04 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:04.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:04 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:04.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:04 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:04.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:04 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:04.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:04 smithi112 ceph-mon[24160]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:04.906 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:04 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:04.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:04 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:04.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:04 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:04.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:04 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:04.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:04 smithi078 ceph-mon[21828]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:06.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:15:06.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:15:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:15:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:15:06.229 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 4 on host 'smithi112' 2024-04-25T04:15:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:06 smithi078 ceph-mon[21828]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:06 smithi078 ceph-mon[21828]: from='osd.4 [v2:172.21.15.112:6800/605605769,v1:172.21.15.112:6801/605605769]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:15:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:06 smithi078 ceph-mon[21828]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:15:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:06 smithi112 ceph-mon[24160]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:06 smithi112 ceph-mon[24160]: from='osd.4 [v2:172.21.15.112:6800/605605769,v1:172.21.15.112:6801/605605769]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:15:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:06 smithi112 ceph-mon[24160]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:15:07.249 DEBUG:teuthology.orchestra.run.smithi112:osd.4> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.4.service 2024-04-25T04:15:07.252 INFO:tasks.cephadm:Deploying osd.5 on smithi112 with /dev/vg_nvme/lv_3... 2024-04-25T04:15:07.252 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-04-25T04:15:08.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-mon[24160]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-25T04:15:08.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-mon[24160]: osdmap e29: 5 total, 4 up, 5 in 2024-04-25T04:15:08.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:08.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-mon[24160]: from='osd.4 [v2:172.21.15.112:6800/605605769,v1:172.21.15.112:6801/605605769]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:08.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-mon[24160]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:08.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-mon[24160]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:08.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:08.327 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[28383]: 2024-04-25T04:15:08.232+0000 7f4c20f90700 -1 osd.4 0 waiting for initial osdmap 2024-04-25T04:15:08.327 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:15:08 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[28383]: 2024-04-25T04:15:08.236+0000 7f4c1a925700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:15:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:08 smithi078 ceph-mon[21828]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-25T04:15:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:08 smithi078 ceph-mon[21828]: osdmap e29: 5 total, 4 up, 5 in 2024-04-25T04:15:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:08 smithi078 ceph-mon[21828]: from='osd.4 [v2:172.21.15.112:6800/605605769,v1:172.21.15.112:6801/605605769]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:08 smithi078 ceph-mon[21828]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:08 smithi078 ceph-mon[21828]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-25T04:15:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:09.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:09 smithi112 ceph-mon[24160]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:15:09.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:09 smithi112 ceph-mon[24160]: osdmap e30: 5 total, 4 up, 5 in 2024-04-25T04:15:09.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:09 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:09.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:09 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:09.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:09 smithi078 ceph-mon[21828]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:15:09.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:09 smithi078 ceph-mon[21828]: osdmap e30: 5 total, 4 up, 5 in 2024-04-25T04:15:09.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:09 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:09.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:09 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:10.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:10 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:15:10.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:10 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:15:10.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:10 smithi112 ceph-mon[24160]: osd.4 [v2:172.21.15.112:6800/605605769,v1:172.21.15.112:6801/605605769] boot 2024-04-25T04:15:10.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:10 smithi112 ceph-mon[24160]: osdmap e31: 5 total, 5 up, 5 in 2024-04-25T04:15:10.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:10.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:10 smithi112 ceph-mon[24160]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:10.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:10 smithi112 ceph-mon[24160]: osdmap e32: 5 total, 5 up, 5 in 2024-04-25T04:15:10.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:10 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:15:10.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:10 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:15:10.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:10 smithi078 ceph-mon[21828]: osd.4 [v2:172.21.15.112:6800/605605769,v1:172.21.15.112:6801/605605769] boot 2024-04-25T04:15:10.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:10 smithi078 ceph-mon[21828]: osdmap e31: 5 total, 5 up, 5 in 2024-04-25T04:15:10.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:15:10.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:10 smithi078 ceph-mon[21828]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:10.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:10 smithi078 ceph-mon[21828]: osdmap e32: 5 total, 5 up, 5 in 2024-04-25T04:15:10.760 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:15:10.780 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_3 2024-04-25T04:15:12.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:12 smithi112 ceph-mon[24160]: osdmap e33: 5 total, 5 up, 5 in 2024-04-25T04:15:12.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:12 smithi112 ceph-mon[24160]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:12.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:12 smithi078 ceph-mon[21828]: osdmap e33: 5 total, 5 up, 5 in 2024-04-25T04:15:12.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:12 smithi078 ceph-mon[21828]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:13.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:13 smithi112 ceph-mon[24160]: osdmap e34: 5 total, 5 up, 5 in 2024-04-25T04:15:13.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:13 smithi078 ceph-mon[21828]: osdmap e34: 5 total, 5 up, 5 in 2024-04-25T04:15:14.501 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: osdmap e35: 5 total, 5 up, 5 in 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:14.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:14.503 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: osdmap e35: 5 total, 5 up, 5 in 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:14.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:14.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:14.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:14.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:15.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:15 smithi112 ceph-mon[24160]: Detected new or changed devices on smithi112 2024-04-25T04:15:15.503 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:15 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi112 to 15936M 2024-04-25T04:15:15.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:15 smithi078 ceph-mon[21828]: Detected new or changed devices on smithi112 2024-04-25T04:15:15.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:15 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi112 to 15936M 2024-04-25T04:15:16.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:16 smithi078 ceph-mon[21828]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-25T04:15:16.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:16 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/3729418102' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d894b71-d7d8-4b39-8112-38335291c870"}]: dispatch 2024-04-25T04:15:16.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:16 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d894b71-d7d8-4b39-8112-38335291c870"}]: dispatch 2024-04-25T04:15:16.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:16 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8d894b71-d7d8-4b39-8112-38335291c870"}]': finished 2024-04-25T04:15:16.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:16 smithi078 ceph-mon[21828]: osdmap e36: 6 total, 5 up, 6 in 2024-04-25T04:15:16.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:16 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:16 smithi112 ceph-mon[24160]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-04-25T04:15:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:16 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/3729418102' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d894b71-d7d8-4b39-8112-38335291c870"}]: dispatch 2024-04-25T04:15:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:16 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8d894b71-d7d8-4b39-8112-38335291c870"}]: dispatch 2024-04-25T04:15:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:16 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8d894b71-d7d8-4b39-8112-38335291c870"}]': finished 2024-04-25T04:15:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:16 smithi112 ceph-mon[24160]: osdmap e36: 6 total, 5 up, 6 in 2024-04-25T04:15:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:16 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:17.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:17 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/184770279' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:15:17.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:17 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/184770279' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:15:18.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:18 smithi078 ceph-mon[21828]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:15:18.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:18 smithi112 ceph-mon[24160]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:15:20.969 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:20 smithi112 ceph-mon[24160]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-25T04:15:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:20 smithi078 ceph-mon[21828]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-25T04:15:22.987 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:22 smithi112 ceph-mon[24160]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-25T04:15:22.987 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T04:15:22.987 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:23.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:22 smithi078 ceph-mon[21828]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-25T04:15:23.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T04:15:23.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:23.882 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:23 smithi112 ceph-mon[24160]: Deploying daemon osd.5 on smithi112 2024-04-25T04:15:24.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:23 smithi078 ceph-mon[21828]: Deploying daemon osd.5 on smithi112 2024-04-25T04:15:25.008 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:24 smithi112 ceph-mon[24160]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-25T04:15:25.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:24 smithi078 ceph-mon[21828]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-25T04:15:26.970 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:26 smithi112 ceph-mon[24160]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:27.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:26 smithi078 ceph-mon[21828]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:28 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:28 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:28 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:28 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:28 smithi078 ceph-mon[21828]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:28.617 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:28 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:28.617 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:28 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:28.617 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:28 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:28.618 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:28 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:28.618 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:28 smithi112 ceph-mon[24160]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:30.211 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 5 on host 'smithi112' 2024-04-25T04:15:30.848 DEBUG:teuthology.orchestra.run.smithi112:osd.5> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.5.service 2024-04-25T04:15:30.850 INFO:tasks.cephadm:Deploying osd.6 on smithi112 with /dev/vg_nvme/lv_2... 2024-04-25T04:15:30.850 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-04-25T04:15:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:30 smithi078 ceph-mon[21828]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:30 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:30 smithi078 ceph-mon[21828]: from='osd.5 [v2:172.21.15.112:6808/2271182165,v1:172.21.15.112:6809/2271182165]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:15:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:30 smithi078 ceph-mon[21828]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:15:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:30 smithi112 ceph-mon[24160]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:30 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:30 smithi112 ceph-mon[24160]: from='osd.5 [v2:172.21.15.112:6808/2271182165,v1:172.21.15.112:6809/2271182165]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:15:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:30 smithi112 ceph-mon[24160]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:15:32.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-mon[24160]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-25T04:15:32.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-mon[24160]: osdmap e37: 6 total, 5 up, 6 in 2024-04-25T04:15:32.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:32.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-mon[24160]: from='osd.5 [v2:172.21.15.112:6808/2271182165,v1:172.21.15.112:6809/2271182165]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:32.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-mon[24160]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:32.560 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-mon[24160]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:32.560 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[31360]: 2024-04-25T04:15:32.214+0000 7f6f232cf700 -1 osd.5 0 waiting for initial osdmap 2024-04-25T04:15:32.560 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:15:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[31360]: 2024-04-25T04:15:32.219+0000 7f6f1c463700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:15:32.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:32 smithi078 ceph-mon[21828]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-25T04:15:32.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:32 smithi078 ceph-mon[21828]: osdmap e37: 6 total, 5 up, 6 in 2024-04-25T04:15:32.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:32 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:32.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:32 smithi078 ceph-mon[21828]: from='osd.5 [v2:172.21.15.112:6808/2271182165,v1:172.21.15.112:6809/2271182165]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:32.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:32 smithi078 ceph-mon[21828]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:32.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:32 smithi078 ceph-mon[21828]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-25T04:15:33.468 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:15:33.468 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: osdmap e38: 6 total, 5 up, 6 in 2024-04-25T04:15:33.468 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:33.468 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:33.468 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:33.468 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: osd.5 [v2:172.21.15.112:6808/2271182165,v1:172.21.15.112:6809/2271182165] boot 2024-04-25T04:15:33.468 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: osdmap e39: 6 total, 6 up, 6 in 2024-04-25T04:15:33.469 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:33 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: osdmap e38: 6 total, 5 up, 6 in 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: osd.5 [v2:172.21.15.112:6808/2271182165,v1:172.21.15.112:6809/2271182165] boot 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: osdmap e39: 6 total, 6 up, 6 in 2024-04-25T04:15:33.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:33 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:15:34.017 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:15:34.034 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_2 2024-04-25T04:15:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:34 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:15:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:34 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:15:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:34 smithi112 ceph-mon[24160]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:34 smithi112 ceph-mon[24160]: osdmap e40: 6 total, 6 up, 6 in 2024-04-25T04:15:34.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:34 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:15:34.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:34 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:15:34.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:34 smithi078 ceph-mon[21828]: pgmap v131: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:34.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:34 smithi078 ceph-mon[21828]: osdmap e40: 6 total, 6 up, 6 in 2024-04-25T04:15:35.485 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:35 smithi112 ceph-mon[24160]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-25T04:15:35.485 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:35 smithi112 ceph-mon[24160]: osdmap e41: 6 total, 6 up, 6 in 2024-04-25T04:15:35.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:35 smithi078 ceph-mon[21828]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-25T04:15:35.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:35 smithi078 ceph-mon[21828]: osdmap e41: 6 total, 6 up, 6 in 2024-04-25T04:15:37.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:36 smithi078 ceph-mon[21828]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:37.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:36 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:37.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:36 smithi078 ceph-mon[21828]: osdmap e42: 6 total, 6 up, 6 in 2024-04-25T04:15:37.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:36 smithi112 ceph-mon[24160]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:37.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:36 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:37.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:36 smithi112 ceph-mon[24160]: osdmap e42: 6 total, 6 up, 6 in 2024-04-25T04:15:38.350 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: Detected new or changed devices on smithi112 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi112 to 7968M 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: osdmap e43: 6 total, 6 up, 6 in 2024-04-25T04:15:38.351 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:15:38.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:15:38.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:15:38.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:38.352 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:38 smithi112 ceph-mon[24160]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: Detected new or changed devices on smithi112 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi112 to 7968M 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:38.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:38.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:38.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: osdmap e43: 6 total, 6 up, 6 in 2024-04-25T04:15:38.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:15:38.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:15:38.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:15:38.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:38.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:38 smithi078 ceph-mon[21828]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:39 smithi112 ceph-mon[24160]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-25T04:15:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:39 smithi112 ceph-mon[24160]: Cluster is now healthy 2024-04-25T04:15:40.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:39 smithi078 ceph-mon[21828]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-25T04:15:40.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:39 smithi078 ceph-mon[21828]: Cluster is now healthy 2024-04-25T04:15:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:40 smithi112 ceph-mon[24160]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-04-25T04:15:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:40 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/2047634134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3864f1f5-f235-40f5-835b-5c04dadb0a5d"}]: dispatch 2024-04-25T04:15:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:40 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3864f1f5-f235-40f5-835b-5c04dadb0a5d"}]: dispatch 2024-04-25T04:15:41.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:40 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3864f1f5-f235-40f5-835b-5c04dadb0a5d"}]': finished 2024-04-25T04:15:41.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:40 smithi112 ceph-mon[24160]: osdmap e44: 7 total, 6 up, 7 in 2024-04-25T04:15:41.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:40 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:41.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:40 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/2992468253' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:15:41.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:40 smithi078 ceph-mon[21828]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-04-25T04:15:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:40 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/2047634134' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3864f1f5-f235-40f5-835b-5c04dadb0a5d"}]: dispatch 2024-04-25T04:15:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:40 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3864f1f5-f235-40f5-835b-5c04dadb0a5d"}]: dispatch 2024-04-25T04:15:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:40 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3864f1f5-f235-40f5-835b-5c04dadb0a5d"}]': finished 2024-04-25T04:15:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:40 smithi078 ceph-mon[21828]: osdmap e44: 7 total, 6 up, 7 in 2024-04-25T04:15:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:40 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:41.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:40 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/2992468253' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:15:42.982 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:42 smithi112 ceph-mon[24160]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:15:43.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:42 smithi078 ceph-mon[21828]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:15:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:44 smithi112 ceph-mon[24160]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-25T04:15:44.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:44 smithi078 ceph-mon[21828]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-25T04:15:47.097 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:46 smithi112 ceph-mon[24160]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-25T04:15:47.097 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:46 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T04:15:47.097 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:46 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:47.097 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:46 smithi112 ceph-mon[24160]: Deploying daemon osd.6 on smithi112 2024-04-25T04:15:47.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:46 smithi078 ceph-mon[21828]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-04-25T04:15:47.107 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:46 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T04:15:47.107 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:46 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:47.107 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:46 smithi078 ceph-mon[21828]: Deploying daemon osd.6 on smithi112 2024-04-25T04:15:49.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:48 smithi078 ceph-mon[21828]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-25T04:15:49.115 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:48 smithi112 ceph-mon[24160]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-25T04:15:50.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:50 smithi112 ceph-mon[24160]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:50.844 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:51.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:50 smithi078 ceph-mon[21828]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:51.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:52.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:52.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:52.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:52.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:51 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:52.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:52.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:15:52.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:15:52.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:51 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:15:53.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:52 smithi078 ceph-mon[21828]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:52 smithi112 ceph-mon[24160]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:54.280 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:54 smithi112 ceph-mon[24160]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:54.280 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:54 smithi112 ceph-mon[24160]: from='osd.6 [v2:172.21.15.112:6816/3900446417,v1:172.21.15.112:6817/3900446417]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:15:54.280 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:54 smithi112 ceph-mon[24160]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:15:54.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:54 smithi078 ceph-mon[21828]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:54.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:54 smithi078 ceph-mon[21828]: from='osd.6 [v2:172.21.15.112:6816/3900446417,v1:172.21.15.112:6817/3900446417]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:15:54.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:54 smithi078 ceph-mon[21828]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:15:54.942 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 6 on host 'smithi112' 2024-04-25T04:15:55.166 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:55 smithi112 ceph-mon[24160]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-25T04:15:55.166 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:55 smithi112 ceph-mon[24160]: osdmap e45: 7 total, 6 up, 7 in 2024-04-25T04:15:55.166 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:55 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:55.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:55 smithi112 ceph-mon[24160]: from='osd.6 [v2:172.21.15.112:6816/3900446417,v1:172.21.15.112:6817/3900446417]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:55.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:55 smithi112 ceph-mon[24160]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:55.167 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:55 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:55.529 DEBUG:teuthology.orchestra.run.smithi112:osd.6> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.6.service 2024-04-25T04:15:55.532 INFO:tasks.cephadm:Deploying osd.7 on smithi112 with /dev/vg_nvme/lv_1... 2024-04-25T04:15:55.532 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-04-25T04:15:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:55 smithi078 ceph-mon[21828]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-25T04:15:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:55 smithi078 ceph-mon[21828]: osdmap e45: 7 total, 6 up, 7 in 2024-04-25T04:15:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:55 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:55 smithi078 ceph-mon[21828]: from='osd.6 [v2:172.21.15.112:6816/3900446417,v1:172.21.15.112:6817/3900446417]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:55 smithi078 ceph-mon[21828]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:15:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:55 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:56.488 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:56 smithi112 ceph-mon[24160]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:15:56.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:56 smithi112 ceph-mon[24160]: osdmap e46: 7 total, 6 up, 7 in 2024-04-25T04:15:56.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:56.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:56.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:56 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:56.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:56 smithi112 ceph-mon[24160]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:56 smithi078 ceph-mon[21828]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:15:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:56 smithi078 ceph-mon[21828]: osdmap e46: 7 total, 6 up, 7 in 2024-04-25T04:15:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:56 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:15:56.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:56 smithi078 ceph-mon[21828]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-25T04:15:57.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:57 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:15:57.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:57 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:15:57.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:57 smithi112 ceph-mon[24160]: osd.6 [v2:172.21.15.112:6816/3900446417,v1:172.21.15.112:6817/3900446417] boot 2024-04-25T04:15:57.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:57 smithi112 ceph-mon[24160]: osdmap e47: 7 total, 7 up, 7 in 2024-04-25T04:15:57.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:57 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:57.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:57 smithi112 ceph-mon[24160]: osdmap e48: 7 total, 7 up, 7 in 2024-04-25T04:15:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:57 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:15:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:57 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:15:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:57 smithi078 ceph-mon[21828]: osd.6 [v2:172.21.15.112:6816/3900446417,v1:172.21.15.112:6817/3900446417] boot 2024-04-25T04:15:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:57 smithi078 ceph-mon[21828]: osdmap e47: 7 total, 7 up, 7 in 2024-04-25T04:15:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:57 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:15:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:57 smithi078 ceph-mon[21828]: osdmap e48: 7 total, 7 up, 7 in 2024-04-25T04:15:58.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:58 smithi078 ceph-mon[21828]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:15:58.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:15:58 smithi078 ceph-mon[21828]: osdmap e49: 7 total, 7 up, 7 in 2024-04-25T04:15:58.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:58 smithi112 ceph-mon[24160]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:15:58.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:15:58 smithi112 ceph-mon[24160]: osdmap e49: 7 total, 7 up, 7 in 2024-04-25T04:15:59.242 INFO:teuthology.orchestra.run.smithi112.stdout: 2024-04-25T04:15:59.262 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi112:vg_nvme/lv_1 2024-04-25T04:16:00.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:00 smithi112 ceph-mon[24160]: osdmap e50: 7 total, 7 up, 7 in 2024-04-25T04:16:00.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:00 smithi112 ceph-mon[24160]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:00.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:00 smithi078 ceph-mon[21828]: osdmap e50: 7 total, 7 up, 7 in 2024-04-25T04:16:00.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:00 smithi078 ceph-mon[21828]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: osdmap e51: 7 total, 7 up, 7 in 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:16:01.510 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:01.511 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:16:01.511 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:01 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: osdmap e51: 7 total, 7 up, 7 in 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:01.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:16:01.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:01 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:02.356 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:02 smithi112 ceph-mon[24160]: Detected new or changed devices on smithi112 2024-04-25T04:16:02.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:02 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi112 to 5312M 2024-04-25T04:16:02.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:02 smithi112 ceph-mon[24160]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T04:16:02.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:02 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:16:02.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:02 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:16:02.357 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:02 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:02.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:02 smithi078 ceph-mon[21828]: Detected new or changed devices on smithi112 2024-04-25T04:16:02.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:02 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi112 to 5312M 2024-04-25T04:16:02.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:02 smithi078 ceph-mon[21828]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-04-25T04:16:02.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:02 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-25T04:16:02.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:02 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-25T04:16:02.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:02 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:03.335 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:03 smithi112 ceph-mon[24160]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:16:03.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:03 smithi078 ceph-mon[21828]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi112:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:16:04.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:04 smithi078 ceph-mon[21828]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:16:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:04 smithi112 ceph-mon[24160]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:16:05.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:05 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/532430194' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7047ec94-c47f-448d-ae4b-79a2a84f7f51"}]: dispatch 2024-04-25T04:16:05.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:05 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7047ec94-c47f-448d-ae4b-79a2a84f7f51"}]: dispatch 2024-04-25T04:16:05.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:05 smithi078 ceph-mon[21828]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7047ec94-c47f-448d-ae4b-79a2a84f7f51"}]': finished 2024-04-25T04:16:05.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:05 smithi078 ceph-mon[21828]: osdmap e52: 8 total, 7 up, 8 in 2024-04-25T04:16:05.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:05.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:05 smithi078 ceph-mon[21828]: from='client.? 172.21.15.112:0/245549270' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:16:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:05 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/532430194' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7047ec94-c47f-448d-ae4b-79a2a84f7f51"}]: dispatch 2024-04-25T04:16:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:05 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7047ec94-c47f-448d-ae4b-79a2a84f7f51"}]: dispatch 2024-04-25T04:16:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:05 smithi112 ceph-mon[24160]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7047ec94-c47f-448d-ae4b-79a2a84f7f51"}]': finished 2024-04-25T04:16:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:05 smithi112 ceph-mon[24160]: osdmap e52: 8 total, 7 up, 8 in 2024-04-25T04:16:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:05 smithi112 ceph-mon[24160]: from='client.? 172.21.15.112:0/245549270' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-25T04:16:06.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:06 smithi078 ceph-mon[21828]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-25T04:16:06.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:16:06.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:16:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:06 smithi112 ceph-mon[24160]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-25T04:16:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:16:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:16:09.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:08 smithi078 ceph-mon[21828]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-25T04:16:09.117 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:08 smithi112 ceph-mon[24160]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-25T04:16:10.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:10 smithi112 ceph-mon[24160]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-25T04:16:10.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T04:16:10.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:11.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:10 smithi078 ceph-mon[21828]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-25T04:16:11.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T04:16:11.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:12.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:11 smithi078 ceph-mon[21828]: Deploying daemon osd.7 on smithi112 2024-04-25T04:16:12.119 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:11 smithi112 ceph-mon[24160]: Deploying daemon osd.7 on smithi112 2024-04-25T04:16:13.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:12 smithi078 ceph-mon[21828]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:13.119 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:12 smithi112 ceph-mon[24160]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:14.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:14 smithi112 ceph-mon[24160]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:14.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:14 smithi078 ceph-mon[21828]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:17.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:16 smithi078 ceph-mon[21828]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:17.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:16 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:17.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:16 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:16:17.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:16 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:17.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:16 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:16:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:16 smithi112 ceph-mon[24160]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:16 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:16 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:16:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:16 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:16 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:16:18.887 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:18 smithi112 ceph-mon[24160]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:19.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:18 smithi078 ceph-mon[21828]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:19.299 INFO:teuthology.orchestra.run.smithi112.stdout:Created osd(s) 7 on host 'smithi112' 2024-04-25T04:16:19.926 DEBUG:teuthology.orchestra.run.smithi112:osd.7> sudo journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.7.service 2024-04-25T04:16:19.929 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-25T04:16:19.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-25T04:16:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:19 smithi078 ceph-mon[21828]: from='osd.7 [v2:172.21.15.112:6824/1678870686,v1:172.21.15.112:6825/1678870686]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T04:16:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:19 smithi078 ceph-mon[21828]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T04:16:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:19 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:19 smithi112 ceph-mon[24160]: from='osd.7 [v2:172.21.15.112:6824/1678870686,v1:172.21.15.112:6825/1678870686]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T04:16:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:19 smithi112 ceph-mon[24160]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T04:16:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:19 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:21.049 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-mon[24160]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:21.049 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-mon[24160]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-25T04:16:21.049 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-mon[24160]: osdmap e53: 8 total, 7 up, 8 in 2024-04-25T04:16:21.049 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:21.049 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-mon[24160]: from='osd.7 [v2:172.21.15.112:6824/1678870686,v1:172.21.15.112:6825/1678870686]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:16:21.049 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-mon[24160]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:16:21.049 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:21.049 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[37424]: 2024-04-25T04:16:20.853+0000 7f40de4c7700 -1 osd.7 0 waiting for initial osdmap 2024-04-25T04:16:21.050 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:16:20 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[37424]: 2024-04-25T04:16:20.860+0000 7f40d9e60700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:16:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:20 smithi078 ceph-mon[21828]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:20 smithi078 ceph-mon[21828]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-25T04:16:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:20 smithi078 ceph-mon[21828]: osdmap e53: 8 total, 7 up, 8 in 2024-04-25T04:16:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:20 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:20 smithi078 ceph-mon[21828]: from='osd.7 [v2:172.21.15.112:6824/1678870686,v1:172.21.15.112:6825/1678870686]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:16:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:20 smithi078 ceph-mon[21828]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:16:21.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:20 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:21.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:21 smithi078 ceph-mon[21828]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:16:21.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:21 smithi078 ceph-mon[21828]: osdmap e54: 8 total, 7 up, 8 in 2024-04-25T04:16:21.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:21.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:21 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:21 smithi112 ceph-mon[24160]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]': finished 2024-04-25T04:16:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:21 smithi112 ceph-mon[24160]: osdmap e54: 8 total, 7 up, 8 in 2024-04-25T04:16:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:21 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:21 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:21.984 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:16:22.727 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1714018581,"num_in_osds":8,"osd_in_since":1714018564,"num_remapped_pgs":1} 2024-04-25T04:16:22.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-25T04:16:22.944 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: purged_snaps scrub starts 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: purged_snaps scrub ok 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: osd.7 [v2:172.21.15.112:6824/1678870686,v1:172.21.15.112:6825/1678870686] boot 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: osdmap e55: 8 total, 8 up, 8 in 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:23.006 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:22 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2954577462' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-25T04:16:23.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: purged_snaps scrub starts 2024-04-25T04:16:23.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: purged_snaps scrub ok 2024-04-25T04:16:23.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-25T04:16:23.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:23.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: osd.7 [v2:172.21.15.112:6824/1678870686,v1:172.21.15.112:6825/1678870686] boot 2024-04-25T04:16:23.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: osdmap e55: 8 total, 8 up, 8 in 2024-04-25T04:16:23.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:16:23.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:22 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2954577462' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-25T04:16:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: osdmap e56: 8 total, 8 up, 8 in 2024-04-25T04:16:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:16:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:23 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: osdmap e56: 8 total, 8 up, 8 in 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:24.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:24.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:16:24.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:16:24.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:16:24.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:23 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:16:24.766 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:16:24.766 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":57,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","created":"2024-04-25T04:10:12.539925+0000","modified":"2024-04-25T04:16:23.816314+0000","last_up_change":"2024-04-25T04:16:21.808104+0000","last_in_change":"2024-04-25T04:16:04.309887+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T04:14:22.027417+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7d61a94e-0810-4c64-a942-e6dad01543d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6803","nonce":2365438107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6805","nonce":2365438107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6809","nonce":2365438107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6807","nonce":2365438107}]},"public_addr":"172.21.15.78:6803/2365438107","cluster_addr":"172.21.15.78:6805/2365438107","heartbeat_back_addr":"172.21.15.78:6809/2365438107","heartbeat_front_addr":"172.21.15.78:6807/2365438107","state":["exists","up"]},{"osd":1,"uuid":"a27d6b5e-7d58-49fd-9f55-896f3d37f68d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6811","nonce":334381174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6813","nonce":334381174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6817","nonce":334381174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6815","nonce":334381174}]},"public_addr":"172.21.15.78:6811/334381174","cluster_addr":"172.21.15.78:6813/334381174","heartbeat_back_addr":"172.21.15.78:6817/334381174","heartbeat_front_addr":"172.21.15.78:6815/334381174","state":["exists","up"]},{"osd":2,"uuid":"c3231645-dee2-4b48-bb4a-28000e933441","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6819","nonce":3901142465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6821","nonce":3901142465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6825","nonce":3901142465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6823","nonce":3901142465}]},"public_addr":"172.21.15.78:6819/3901142465","cluster_addr":"172.21.15.78:6821/3901142465","heartbeat_back_addr":"172.21.15.78:6825/3901142465","heartbeat_front_addr":"172.21.15.78:6823/3901142465","state":["exists","up"]},{"osd":3,"uuid":"3cc63613-0ceb-4585-aaf8-03283211d41c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6827","nonce":2527808235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6829","nonce":2527808235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6833","nonce":2527808235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6831","nonce":2527808235}]},"public_addr":"172.21.15.78:6827/2527808235","cluster_addr":"172.21.15.78:6829/2527808235","heartbeat_back_addr":"172.21.15.78:6833/2527808235","heartbeat_front_addr":"172.21.15.78:6831/2527808235","state":["exists","up"]},{"osd":4,"uuid":"bb4c070a-b7ec-4753-b794-847844fa7604","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6801","nonce":605605769}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6803","nonce":605605769}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6807","nonce":605605769}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6805","nonce":605605769}]},"public_addr":"172.21.15.112:6801/605605769","cluster_addr":"172.21.15.112:6803/605605769","heartbeat_back_addr":"172.21.15.112:6807/605605769","heartbeat_front_addr":"172.21.15.112:6805/605605769","state":["exists","up"]},{"osd":5,"uuid":"8d894b71-d7d8-4b39-8112-38335291c870","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6809","nonce":2271182165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6811","nonce":2271182165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6815","nonce":2271182165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6813","nonce":2271182165}]},"public_addr":"172.21.15.112:6809/2271182165","cluster_addr":"172.21.15.112:6811/2271182165","heartbeat_back_addr":"172.21.15.112:6815/2271182165","heartbeat_front_addr":"172.21.15.112:6813/2271182165","state":["exists","up"]},{"osd":6,"uuid":"3864f1f5-f235-40f5-835b-5c04dadb0a5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6817","nonce":3900446417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6819","nonce":3900446417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6823","nonce":3900446417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6821","nonce":3900446417}]},"public_addr":"172.21.15.112:6817/3900446417","cluster_addr":"172.21.15.112:6819/3900446417","heartbeat_back_addr":"172.21.15.112:6823/3900446417","heartbeat_front_addr":"172.21.15.112:6821/3900446417","state":["exists","up"]},{"osd":7,"uuid":"7047ec94-c47f-448d-ae4b-79a2a84f7f51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6825","nonce":1678870686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6827","nonce":1678870686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6831","nonce":1678870686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6829","nonce":1678870686}]},"public_addr":"172.21.15.112:6825/1678870686","cluster_addr":"172.21.15.112:6827/1678870686","heartbeat_back_addr":"172.21.15.112:6831/1678870686","heartbeat_front_addr":"172.21.15.112:6829/1678870686","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:13:20.638192+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:13:51.754504+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:14:18.864672+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:14:46.798048+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:07.654202+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:31.288409+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:54.934976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:16:19.878126+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.78:6801/1961539517":"2024-04-26T04:12:05.668530+0000","172.21.15.78:6800/1961539517":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/3957863029":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/3866245432":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/927861512":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/4050624330":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6801/3886928174":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6800/3886928174":"2024-04-26T04:11:08.802622+0000","172.21.15.78:0/3388697634":"2024-04-26T04:10:39.116297+0000","172.21.15.78:6800/2631949670":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/545268089":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6801/2631949670":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/1474296960":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/136448278":"2024-04-26T04:11:08.802622+0000","172.21.15.78:0/994343344":"2024-04-26T04:10:39.116297+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T04:16:25.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:24 smithi078 ceph-mon[21828]: Detected new or changed devices on smithi112 2024-04-25T04:16:25.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:24 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi112 to 3984M 2024-04-25T04:16:25.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:24 smithi078 ceph-mon[21828]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:25.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:24 smithi078 ceph-mon[21828]: osdmap e57: 8 total, 8 up, 8 in 2024-04-25T04:16:25.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:24 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/907597420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T04:16:25.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:24 smithi112 ceph-mon[24160]: Detected new or changed devices on smithi112 2024-04-25T04:16:25.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:24 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi112 to 3984M 2024-04-25T04:16:25.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:24 smithi112 ceph-mon[24160]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:25.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:24 smithi112 ceph-mon[24160]: osdmap e57: 8 total, 8 up, 8 in 2024-04-25T04:16:25.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:24 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/907597420' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T04:16:25.469 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-25T04:14:22.027417+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-25T04:16:25.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-25T04:16:25.683 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:26.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:25 smithi078 ceph-mon[21828]: osdmap e58: 8 total, 8 up, 8 in 2024-04-25T04:16:26.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:25 smithi112 ceph-mon[24160]: osdmap e58: 8 total, 8 up, 8 in 2024-04-25T04:16:26.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:26 smithi078 ceph-mon[21828]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:26.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:26 smithi078 ceph-mon[21828]: osdmap e59: 8 total, 8 up, 8 in 2024-04-25T04:16:27.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:26 smithi112 ceph-mon[24160]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:27.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:26 smithi112 ceph-mon[24160]: osdmap e59: 8 total, 8 up, 8 in 2024-04-25T04:16:27.343 INFO:teuthology.orchestra.run.smithi078.stdout:pg_num: 1 2024-04-25T04:16:27.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:27 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/437921910' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T04:16:27.998 INFO:tasks.cephadm:Setting up client nodes... 2024-04-25T04:16:27.999 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-25T04:16:28.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:27 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/437921910' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-25T04:16:28.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:28 smithi078 ceph-mon[21828]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:29.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:28 smithi112 ceph-mon[24160]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:29.998 INFO:teuthology.orchestra.run.smithi078.stdout:[client.0] 2024-04-25T04:16:29.998 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQAd2SlmSJD+OhAAXWPtjL8Z6cb0aJFHOC565A== 2024-04-25T04:16:30.724 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-04-25T04:16:30.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-25T04:16:30.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-25T04:16:30.762 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-25T04:16:30.762 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-25T04:16:30.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-25T04:16:31.012 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:30 smithi078 ceph-mon[21828]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:16:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:30 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2928127896' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-25T04:16:31.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:30 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2928127896' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-25T04:16:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:30 smithi112 ceph-mon[24160]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-25T04:16:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:30 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2928127896' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-25T04:16:31.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:30 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2928127896' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-25T04:16:32.957 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:16:33.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:32 smithi078 ceph-mon[21828]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-25T04:16:33.118 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:32 smithi112 ceph-mon[24160]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-25T04:16:33.659 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi078.eugwik","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":2211075934},{"type":"v1","addr":"172.21.15.78:6801","nonce":2211075934}]},"active_addr":"172.21.15.78:6801/2211075934","active_change":"2024-04-25T04:12:05.668817+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14226,"name":"smithi112.lvtlnw","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1849371061}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2371911336}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1908137830}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":474013235}]}]}} 2024-04-25T04:16:33.665 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-25T04:16:33.665 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-25T04:16:33.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-25T04:16:33.876 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:33.998 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:33 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1295128482' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-25T04:16:34.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:33 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/1295128482' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-25T04:16:34.999 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:34 smithi078 ceph-mon[21828]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T04:16:35.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:34 smithi112 ceph-mon[24160]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-25T04:16:35.603 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:16:35.603 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","created":"2024-04-25T04:10:12.539925+0000","modified":"2024-04-25T04:16:25.828124+0000","last_up_change":"2024-04-25T04:16:21.808104+0000","last_in_change":"2024-04-25T04:16:04.309887+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T04:14:22.027417+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7d61a94e-0810-4c64-a942-e6dad01543d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6803","nonce":2365438107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6805","nonce":2365438107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6809","nonce":2365438107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6807","nonce":2365438107}]},"public_addr":"172.21.15.78:6803/2365438107","cluster_addr":"172.21.15.78:6805/2365438107","heartbeat_back_addr":"172.21.15.78:6809/2365438107","heartbeat_front_addr":"172.21.15.78:6807/2365438107","state":["exists","up"]},{"osd":1,"uuid":"a27d6b5e-7d58-49fd-9f55-896f3d37f68d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6811","nonce":334381174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6813","nonce":334381174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6817","nonce":334381174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6815","nonce":334381174}]},"public_addr":"172.21.15.78:6811/334381174","cluster_addr":"172.21.15.78:6813/334381174","heartbeat_back_addr":"172.21.15.78:6817/334381174","heartbeat_front_addr":"172.21.15.78:6815/334381174","state":["exists","up"]},{"osd":2,"uuid":"c3231645-dee2-4b48-bb4a-28000e933441","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6819","nonce":3901142465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6821","nonce":3901142465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6825","nonce":3901142465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6823","nonce":3901142465}]},"public_addr":"172.21.15.78:6819/3901142465","cluster_addr":"172.21.15.78:6821/3901142465","heartbeat_back_addr":"172.21.15.78:6825/3901142465","heartbeat_front_addr":"172.21.15.78:6823/3901142465","state":["exists","up"]},{"osd":3,"uuid":"3cc63613-0ceb-4585-aaf8-03283211d41c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6827","nonce":2527808235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6829","nonce":2527808235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6833","nonce":2527808235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6831","nonce":2527808235}]},"public_addr":"172.21.15.78:6827/2527808235","cluster_addr":"172.21.15.78:6829/2527808235","heartbeat_back_addr":"172.21.15.78:6833/2527808235","heartbeat_front_addr":"172.21.15.78:6831/2527808235","state":["exists","up"]},{"osd":4,"uuid":"bb4c070a-b7ec-4753-b794-847844fa7604","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6801","nonce":605605769}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6803","nonce":605605769}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6807","nonce":605605769}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6805","nonce":605605769}]},"public_addr":"172.21.15.112:6801/605605769","cluster_addr":"172.21.15.112:6803/605605769","heartbeat_back_addr":"172.21.15.112:6807/605605769","heartbeat_front_addr":"172.21.15.112:6805/605605769","state":["exists","up"]},{"osd":5,"uuid":"8d894b71-d7d8-4b39-8112-38335291c870","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6809","nonce":2271182165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6811","nonce":2271182165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6815","nonce":2271182165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6813","nonce":2271182165}]},"public_addr":"172.21.15.112:6809/2271182165","cluster_addr":"172.21.15.112:6811/2271182165","heartbeat_back_addr":"172.21.15.112:6815/2271182165","heartbeat_front_addr":"172.21.15.112:6813/2271182165","state":["exists","up"]},{"osd":6,"uuid":"3864f1f5-f235-40f5-835b-5c04dadb0a5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6817","nonce":3900446417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6819","nonce":3900446417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6823","nonce":3900446417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6821","nonce":3900446417}]},"public_addr":"172.21.15.112:6817/3900446417","cluster_addr":"172.21.15.112:6819/3900446417","heartbeat_back_addr":"172.21.15.112:6823/3900446417","heartbeat_front_addr":"172.21.15.112:6821/3900446417","state":["exists","up"]},{"osd":7,"uuid":"7047ec94-c47f-448d-ae4b-79a2a84f7f51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6825","nonce":1678870686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6827","nonce":1678870686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6831","nonce":1678870686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6829","nonce":1678870686}]},"public_addr":"172.21.15.112:6825/1678870686","cluster_addr":"172.21.15.112:6827/1678870686","heartbeat_back_addr":"172.21.15.112:6831/1678870686","heartbeat_front_addr":"172.21.15.112:6829/1678870686","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:13:20.638192+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:13:51.754504+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:14:18.864672+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:14:46.798048+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:07.654202+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:31.288409+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:54.934976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:16:19.878126+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6801/1961539517":"2024-04-26T04:12:05.668530+0000","172.21.15.78:6800/1961539517":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/3957863029":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/3866245432":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/927861512":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/4050624330":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6801/3886928174":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6800/3886928174":"2024-04-26T04:11:08.802622+0000","172.21.15.78:0/3388697634":"2024-04-26T04:10:39.116297+0000","172.21.15.78:6800/2631949670":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/545268089":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6801/2631949670":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/1474296960":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/136448278":"2024-04-26T04:11:08.802622+0000","172.21.15.78:0/994343344":"2024-04-26T04:10:39.116297+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T04:16:36.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:35 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/1909970561' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T04:16:36.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:35 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1909970561' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T04:16:36.342 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-25T04:16:36.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-25T04:16:36.558 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:37.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:36 smithi078 ceph-mon[21828]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-25T04:16:37.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:36 smithi112 ceph-mon[24160]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-25T04:16:38.292 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:16:38.292 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"9a8d9f72-02b9-11ef-bc93-c7b262605968","created":"2024-04-25T04:10:12.539925+0000","modified":"2024-04-25T04:16:25.828124+0000","last_up_change":"2024-04-25T04:16:21.808104+0000","last_in_change":"2024-04-25T04:16:04.309887+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-25T04:14:22.027417+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"7d61a94e-0810-4c64-a942-e6dad01543d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6803","nonce":2365438107}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6805","nonce":2365438107}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6809","nonce":2365438107}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2365438107},{"type":"v1","addr":"172.21.15.78:6807","nonce":2365438107}]},"public_addr":"172.21.15.78:6803/2365438107","cluster_addr":"172.21.15.78:6805/2365438107","heartbeat_back_addr":"172.21.15.78:6809/2365438107","heartbeat_front_addr":"172.21.15.78:6807/2365438107","state":["exists","up"]},{"osd":1,"uuid":"a27d6b5e-7d58-49fd-9f55-896f3d37f68d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6811","nonce":334381174}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6813","nonce":334381174}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6817","nonce":334381174}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":334381174},{"type":"v1","addr":"172.21.15.78:6815","nonce":334381174}]},"public_addr":"172.21.15.78:6811/334381174","cluster_addr":"172.21.15.78:6813/334381174","heartbeat_back_addr":"172.21.15.78:6817/334381174","heartbeat_front_addr":"172.21.15.78:6815/334381174","state":["exists","up"]},{"osd":2,"uuid":"c3231645-dee2-4b48-bb4a-28000e933441","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6819","nonce":3901142465}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6821","nonce":3901142465}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6825","nonce":3901142465}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":3901142465},{"type":"v1","addr":"172.21.15.78:6823","nonce":3901142465}]},"public_addr":"172.21.15.78:6819/3901142465","cluster_addr":"172.21.15.78:6821/3901142465","heartbeat_back_addr":"172.21.15.78:6825/3901142465","heartbeat_front_addr":"172.21.15.78:6823/3901142465","state":["exists","up"]},{"osd":3,"uuid":"3cc63613-0ceb-4585-aaf8-03283211d41c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6827","nonce":2527808235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6829","nonce":2527808235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6833","nonce":2527808235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":2527808235},{"type":"v1","addr":"172.21.15.78:6831","nonce":2527808235}]},"public_addr":"172.21.15.78:6827/2527808235","cluster_addr":"172.21.15.78:6829/2527808235","heartbeat_back_addr":"172.21.15.78:6833/2527808235","heartbeat_front_addr":"172.21.15.78:6831/2527808235","state":["exists","up"]},{"osd":4,"uuid":"bb4c070a-b7ec-4753-b794-847844fa7604","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6800","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6801","nonce":605605769}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6802","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6803","nonce":605605769}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6806","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6807","nonce":605605769}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6804","nonce":605605769},{"type":"v1","addr":"172.21.15.112:6805","nonce":605605769}]},"public_addr":"172.21.15.112:6801/605605769","cluster_addr":"172.21.15.112:6803/605605769","heartbeat_back_addr":"172.21.15.112:6807/605605769","heartbeat_front_addr":"172.21.15.112:6805/605605769","state":["exists","up"]},{"osd":5,"uuid":"8d894b71-d7d8-4b39-8112-38335291c870","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6808","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6809","nonce":2271182165}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6810","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6811","nonce":2271182165}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6814","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6815","nonce":2271182165}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6812","nonce":2271182165},{"type":"v1","addr":"172.21.15.112:6813","nonce":2271182165}]},"public_addr":"172.21.15.112:6809/2271182165","cluster_addr":"172.21.15.112:6811/2271182165","heartbeat_back_addr":"172.21.15.112:6815/2271182165","heartbeat_front_addr":"172.21.15.112:6813/2271182165","state":["exists","up"]},{"osd":6,"uuid":"3864f1f5-f235-40f5-835b-5c04dadb0a5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6816","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6817","nonce":3900446417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6818","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6819","nonce":3900446417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6822","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6823","nonce":3900446417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6820","nonce":3900446417},{"type":"v1","addr":"172.21.15.112:6821","nonce":3900446417}]},"public_addr":"172.21.15.112:6817/3900446417","cluster_addr":"172.21.15.112:6819/3900446417","heartbeat_back_addr":"172.21.15.112:6823/3900446417","heartbeat_front_addr":"172.21.15.112:6821/3900446417","state":["exists","up"]},{"osd":7,"uuid":"7047ec94-c47f-448d-ae4b-79a2a84f7f51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6824","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6825","nonce":1678870686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6826","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6827","nonce":1678870686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6830","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6831","nonce":1678870686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.112:6828","nonce":1678870686},{"type":"v1","addr":"172.21.15.112:6829","nonce":1678870686}]},"public_addr":"172.21.15.112:6825/1678870686","cluster_addr":"172.21.15.112:6827/1678870686","heartbeat_back_addr":"172.21.15.112:6831/1678870686","heartbeat_front_addr":"172.21.15.112:6829/1678870686","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:13:20.638192+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:13:51.754504+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:14:18.864672+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:14:46.798048+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:07.654202+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:31.288409+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:15:54.934976+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-25T04:16:19.878126+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6801/1961539517":"2024-04-26T04:12:05.668530+0000","172.21.15.78:6800/1961539517":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/3957863029":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/3866245432":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/927861512":"2024-04-26T04:12:05.668530+0000","172.21.15.78:0/4050624330":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6801/3886928174":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6800/3886928174":"2024-04-26T04:11:08.802622+0000","172.21.15.78:0/3388697634":"2024-04-26T04:10:39.116297+0000","172.21.15.78:6800/2631949670":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/545268089":"2024-04-26T04:11:08.802622+0000","172.21.15.78:6801/2631949670":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/1474296960":"2024-04-26T04:10:39.116297+0000","172.21.15.78:0/136448278":"2024-04-26T04:11:08.802622+0000","172.21.15.78:0/994343344":"2024-04-26T04:10:39.116297+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-25T04:16:39.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-25T04:16:39.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-25T04:16:39.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-25T04:16:39.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-25T04:16:39.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-25T04:16:39.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-25T04:16:39.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-25T04:16:39.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-25T04:16:39.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:38 smithi078 ceph-mon[21828]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-25T04:16:39.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:38 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1611902039' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T04:16:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:38 smithi112 ceph-mon[24160]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-25T04:16:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:38 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/1611902039' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-25T04:16:39.514 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:39.517 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:39.517 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:39.517 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:39.518 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:39.521 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:39.521 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:39.522 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:40 smithi112 ceph-mon[24160]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-25T04:16:41.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:40 smithi078 ceph-mon[21828]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-25T04:16:43.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:42 smithi078 ceph-mon[21828]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:42 smithi112 ceph-mon[24160]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:44.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:44 smithi078 ceph-mon[21828]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:44 smithi112 ceph-mon[24160]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:46.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:46 smithi078 ceph-mon[21828]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:47.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:46 smithi112 ceph-mon[24160]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:49.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:48 smithi078 ceph-mon[21828]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:49.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:48 smithi112 ceph-mon[24160]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:50.786 INFO:teuthology.orchestra.run.smithi078.stdout:133143986198 2024-04-25T04:16:50.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-25T04:16:51.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:50 smithi078 ceph-mon[21828]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:51.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:50 smithi112 ceph-mon[24160]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:51.985 INFO:teuthology.orchestra.run.smithi078.stdout:81604378657 2024-04-25T04:16:51.985 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-25T04:16:52.980 INFO:teuthology.orchestra.run.smithi078.stdout:201863462924 2024-04-25T04:16:52.980 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-25T04:16:53.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:52 smithi078 ceph-mon[21828]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:52 smithi112 ceph-mon[24160]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:54.333 INFO:teuthology.orchestra.run.smithi078.stdout:38654705709 2024-04-25T04:16:54.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-25T04:16:54.445 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:54 smithi078 ceph-mon[21828]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:54 smithi112 ceph-mon[24160]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:54.741 INFO:teuthology.orchestra.run.smithi078.stdout:111669149724 2024-04-25T04:16:54.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-25T04:16:55.900 INFO:teuthology.orchestra.run.smithi078.stdout:60129542182 2024-04-25T04:16:55.901 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-25T04:16:56.217 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:56.217 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:56.218 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:56.218 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:56.221 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:16:56.226 INFO:teuthology.orchestra.run.smithi078.stdout:167503724562 2024-04-25T04:16:56.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-25T04:16:56.600 INFO:teuthology.orchestra.run.smithi078.stdout:236223201289 2024-04-25T04:16:56.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-25T04:16:57.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:56 smithi078 ceph-mon[21828]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:57.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:56 smithi112 ceph-mon[24160]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:59.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:16:58 smithi078 ceph-mon[21828]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:16:59.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:16:58 smithi112 ceph-mon[24160]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:01.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:00 smithi078 ceph-mon[21828]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:01.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:00 smithi112 ceph-mon[24160]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:01.570 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:17:01.771 INFO:teuthology.orchestra.run.smithi078.stdout:133143986201 2024-04-25T04:17:01.971 INFO:teuthology.orchestra.run.smithi078.stdout:38654705711 2024-04-25T04:17:02.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:01 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/952983695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-25T04:17:02.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:01 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/952983695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-25T04:17:02.439 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:17:02.440 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:17:02.582 INFO:teuthology.orchestra.run.smithi078.stdout:201863462927 2024-04-25T04:17:02.928 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705711 for osd.0 2024-04-25T04:17:02.928 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:03.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:02 smithi078 ceph-mon[21828]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:03.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:02 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1049395877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-25T04:17:03.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:02 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2980571985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-25T04:17:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:02 smithi112 ceph-mon[24160]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:02 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/1049395877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-25T04:17:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:02 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2980571985' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-25T04:17:04.054 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986198 got 133143986201 for osd.4 2024-04-25T04:17:04.054 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:04.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:04 smithi078 ceph-mon[21828]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:04 smithi112 ceph-mon[24160]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:06.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:17:06.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:17:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:17:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:17:06.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:06 smithi078 ceph-mon[21828]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:06.920 INFO:teuthology.orchestra.run.smithi078.stdout:236223201292 2024-04-25T04:17:07.128 INFO:teuthology.orchestra.run.smithi078.stdout:81604378660 2024-04-25T04:17:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:06 smithi112 ceph-mon[24160]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:07.518 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-04-25T04:17:07.518 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:07.897 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462924 got 201863462927 for osd.6 2024-04-25T04:17:07.897 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:08.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:07 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/961062231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-25T04:17:08.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:07 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/308160178' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-25T04:17:08.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:07 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/961062231' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-25T04:17:08.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:07 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/308160178' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-25T04:17:09.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:08 smithi078 ceph-mon[21828]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:09.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:08 smithi112 ceph-mon[24160]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:09.492 INFO:teuthology.orchestra.run.smithi078.stdout:167503724564 2024-04-25T04:17:09.950 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378660 for osd.2 2024-04-25T04:17:09.950 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:10.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:09 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/452704025' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-25T04:17:10.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:09 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/452704025' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-25T04:17:11.006 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724562 got 167503724564 for osd.5 2024-04-25T04:17:11.006 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:11.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:10 smithi112 ceph-mon[24160]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:11.176 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:10 smithi078 ceph-mon[21828]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:11.382 INFO:teuthology.orchestra.run.smithi078.stdout:111669149728 2024-04-25T04:17:11.965 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:11 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2793379911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-25T04:17:11.988 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149728 for osd.3 2024-04-25T04:17:11.988 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:12.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:11 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2793379911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-25T04:17:12.465 INFO:teuthology.orchestra.run.smithi078.stdout:60129542186 2024-04-25T04:17:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:12 smithi112 ceph-mon[24160]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:12 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2655108151' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-25T04:17:13.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:12 smithi078 ceph-mon[21828]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:13.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:12 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2655108151' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-25T04:17:13.260 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542186 for osd.1 2024-04-25T04:17:13.260 DEBUG:teuthology.parallel:result is None 2024-04-25T04:17:13.260 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-25T04:17:13.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-25T04:17:13.474 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:17:14.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:14 smithi078 ceph-mon[21828]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:14.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:14 smithi112 ceph-mon[24160]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:15.230 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:17:15.231 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-04-25T04:17:15.848 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-04-25T04:17:13.774003+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48864,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912352,"statfs":{"total":767960285184,"available":767910248448,"internally_reserved":0,"allocated":6455296,"data_stored":3546884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001895"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-25T04:16:25.839904+0000","last_change":"2024-04-25T04:16:25.839904+0000","last_active":"2024-04-25T04:16:25.839904+0000","last_peered":"2024-04-25T04:16:25.839904+0000","last_clean":"2024-04-25T04:16:25.839904+0000","last_became_active":"2024-04-25T04:16:25.839431+0000","last_became_peered":"2024-04-25T04:16:25.839431+0000","last_unstale":"2024-04-25T04:16:25.839904+0000","last_undegraded":"2024-04-25T04:16:25.839904+0000","last_fullsized":"2024-04-25T04:16:25.839904+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T04:14:22.027866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T04:14:22.027866+0000","last_clean_scrub_stamp":"2024-04-25T04:14:22.027866+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:46:30.448874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57399999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33900000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42499999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462929,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.50800000000000001}]},{"osd":1,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.45100000000000001}]},{"osd":2,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.60799999999999998}]},{"osd":3,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59399999999999997}]},{"osd":4,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.57399999999999995}]},{"osd":5,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.624},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.72299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542186,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:17:02 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.311},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.46899999999999997,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.376},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.31900000000000001},"min":{"1min":0.193,"5min":0.19,"15min":0.19},"max":{"1min":0.54500000000000004,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.33900000000000002}]},{"osd":2,"last update":"Thu Apr 25 04:16:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36899999999999999,"15min":0.36099999999999999},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.373,"15min":0.36799999999999999},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51300000000000001,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.36199999999999999}]},{"osd":3,"last update":"Thu Apr 25 04:16:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59999999999999998,"5min":0.621,"15min":0.621},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.39300000000000002}]},{"osd":4,"last update":"Thu Apr 25 04:16:10 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.77100000000000002}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.59999999999999998}]},{"osd":6,"last update":"Thu Apr 25 04:16:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.76000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67700000000000005}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6948,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738204,"statfs":{"total":95995035648,"available":95987920896,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Apr 25 04:17:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.33000000000000002,"15min":0.308},"min":{"1min":0.248,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.33300000000000002,"15min":0.30199999999999999},"min":{"1min":0.26100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.52900000000000003}]},{"osd":2,"last update":"Thu Apr 25 04:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35499999999999998,"15min":0.34899999999999998},"min":{"1min":0.26600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.48699999999999999,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.38,"15min":0.376},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.59999999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Apr 25 04:16:55 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.438,"15min":0.42899999999999999},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.438,"15min":0.433},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.56000000000000005}]},{"osd":4,"last update":"Thu Apr 25 04:17:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.42399999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.59399999999999997,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.376,"15min":0.376},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.624}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.71699999999999997}]},{"osd":6,"last update":"Thu Apr 25 04:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.751},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.69899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74299999999999999}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:27 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.372,"15min":0.36599999999999999},"min":{"1min":0.27400000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.314},{"interface":"front","average":{"1min":0.435,"5min":0.38,"15min":0.371},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.32800000000000001}]},{"osd":1,"last update":"Thu Apr 25 04:16:27 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.40600000000000003,"15min":0.39600000000000002},"min":{"1min":0.312,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.439,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.314,"5min":0.248,"15min":0.248},"max":{"1min":0.59499999999999997,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.27700000000000002}]},{"osd":3,"last update":"Thu Apr 25 04:16:50 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.45100000000000001,"15min":0.44500000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.23,"5min":1.23,"15min":1.23},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.30199999999999999}]},{"osd":4,"last update":"Thu Apr 25 04:16:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.502}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.40699999999999997}]},{"osd":6,"last update":"Thu Apr 25 04:17:02 2024","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.41899999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.39100000000000001,"15min":0.39500000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51400000000000001,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.37,"5min":0.39500000000000002,"15min":0.39900000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.503,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.38900000000000001}]},{"osd":1,"last update":"Thu Apr 25 04:16:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.41399999999999998,"15min":0.41699999999999998},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.47899999999999998,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.318},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40000000000000002,"15min":0.40600000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.46300000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.40500000000000003}]},{"osd":2,"last update":"Thu Apr 25 04:16:53 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.44500000000000001,"15min":0.45000000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.54900000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.42199999999999999,"15min":0.42699999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.41499999999999998}]},{"osd":4,"last update":"Thu Apr 25 04:16:11 2024","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":6.2409999999999997,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.50600000000000001}]},{"osd":5,"last update":"Thu Apr 25 04:16:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.628},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.65700000000000003}]},{"osd":6,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.58599999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64600000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986203,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:11 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.67300000000000004}]},{"osd":1,"last update":"Thu Apr 25 04:16:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.63500000000000001}]},{"osd":2,"last update":"Thu Apr 25 04:16:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.081,"5min":1.081,"15min":1.081},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.60699999999999998}]},{"osd":3,"last update":"Thu Apr 25 04:16:11 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.53300000000000003}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.68700000000000006}]},{"osd":6,"last update":"Thu Apr 25 04:17:00 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.65600000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]}]},{"osd":5,"up_from":39,"seq":167503724566,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.46000000000000002}]},{"osd":1,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.503},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.53000000000000003}]},{"osd":2,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.63600000000000001}]},{"osd":3,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.56200000000000006}]},{"osd":4,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.65000000000000002}]},{"osd":6,"last update":"Thu Apr 25 04:16:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.58099999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T04:17:15.850 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-25T04:17:16.058 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:17:17.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:16 smithi078 ceph-mon[21828]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:17.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:16 smithi078 ceph-mon[21828]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:16 smithi112 ceph-mon[24160]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:16 smithi112 ceph-mon[24160]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:17.784 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:17:17.785 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-04-25T04:17:18.471 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-04-25T04:17:17.774800+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48864,"kb_used_data":6304,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912352,"statfs":{"total":767960285184,"available":767910248448,"internally_reserved":0,"allocated":6455296,"data_stored":3546884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002089"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-25T04:16:25.839904+0000","last_change":"2024-04-25T04:16:25.839904+0000","last_active":"2024-04-25T04:16:25.839904+0000","last_peered":"2024-04-25T04:16:25.839904+0000","last_clean":"2024-04-25T04:16:25.839904+0000","last_became_active":"2024-04-25T04:16:25.839431+0000","last_became_peered":"2024-04-25T04:16:25.839431+0000","last_unstale":"2024-04-25T04:16:25.839904+0000","last_undegraded":"2024-04-25T04:16:25.839904+0000","last_fullsized":"2024-04-25T04:16:25.839904+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-25T04:14:22.027866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-25T04:14:22.027866+0000","last_clean_scrub_stamp":"2024-04-25T04:14:22.027866+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-26T07:46:30.448874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57399999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33900000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42499999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.54400000000000004}]},{"osd":1,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.53000000000000003}]},{"osd":2,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.496},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.58899999999999997}]},{"osd":3,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.46300000000000002}]},{"osd":4,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.39900000000000002}]},{"osd":5,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:17:02 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.311},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.46899999999999997,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.34100000000000003,"15min":0.31900000000000001},"min":{"1min":0.193,"5min":0.19,"15min":0.19},"max":{"1min":0.54500000000000004,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.376}]},{"osd":2,"last update":"Thu Apr 25 04:16:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36899999999999999,"15min":0.36099999999999999},"min":{"1min":0.27600000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.438},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.373,"15min":0.36799999999999999},"min":{"1min":0.26200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51300000000000001,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.36199999999999999}]},{"osd":3,"last update":"Thu Apr 25 04:16:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.59999999999999998,"5min":0.621,"15min":0.621},"last":0.5},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.33000000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.42599999999999999}]},{"osd":4,"last update":"Thu Apr 25 04:17:10 2024","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":0.70799999999999996,"15min":0.63900000000000001},"min":{"1min":0.45700000000000002,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.6699999999999999,"5min":9.6699999999999999,"15min":9.6699999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.624,"15min":0.625},"min":{"1min":0.49399999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.77100000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.64500000000000002}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.73199999999999998}]},{"osd":6,"last update":"Thu Apr 25 04:16:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.80700000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77600000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6948,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738204,"statfs":{"total":95995035648,"available":95987920896,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Apr 25 04:17:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.33000000000000002,"15min":0.308},"min":{"1min":0.248,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.33300000000000002,"15min":0.30199999999999999},"min":{"1min":0.26100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.52900000000000003}]},{"osd":2,"last update":"Thu Apr 25 04:16:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35499999999999998,"15min":0.34899999999999998},"min":{"1min":0.26600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.48699999999999999,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.38,"15min":0.376},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.59999999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Apr 25 04:16:55 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.438,"15min":0.42899999999999999},"min":{"1min":0.33300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.438,"15min":0.433},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.1220000000000001,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.56000000000000005}]},{"osd":4,"last update":"Thu Apr 25 04:17:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.55200000000000005,"15min":0.53600000000000003},"min":{"1min":0.42399999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.59399999999999997,"15min":0.58499999999999996},"min":{"1min":0.47599999999999998,"5min":0.376,"15min":0.376},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.624}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.71699999999999997}]},{"osd":6,"last update":"Thu Apr 25 04:16:58 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.751},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.69899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74299999999999999}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:27 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.372,"15min":0.36599999999999999},"min":{"1min":0.27400000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.38,"15min":0.371},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.39900000000000002}]},{"osd":1,"last update":"Thu Apr 25 04:16:27 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.40600000000000003,"15min":0.39600000000000002},"min":{"1min":0.312,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.39000000000000001,"15min":0.38200000000000001},"min":{"1min":0.314,"5min":0.248,"15min":0.248},"max":{"1min":0.59499999999999997,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.28199999999999997}]},{"osd":3,"last update":"Thu Apr 25 04:16:50 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.45100000000000001,"15min":0.44500000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.23,"5min":1.23,"15min":1.23},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.35499999999999998}]},{"osd":4,"last update":"Thu Apr 25 04:17:12 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.61299999999999999,"15min":0.59799999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.8859999999999999,"5min":1.8859999999999999,"15min":1.8859999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.63800000000000001,"15min":0.63300000000000001},"min":{"1min":0.47699999999999998,"5min":0.433,"15min":0.433},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.59699999999999998}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.48099999999999998}]},{"osd":6,"last update":"Thu Apr 25 04:17:02 2024","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.61599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.39100000000000001,"15min":0.39500000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51400000000000001,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.37,"5min":0.39500000000000002,"15min":0.39900000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.503,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.39600000000000002}]},{"osd":1,"last update":"Thu Apr 25 04:16:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.41399999999999998,"15min":0.41699999999999998},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.47899999999999998,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.40000000000000002,"15min":0.40600000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.46300000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Apr 25 04:16:53 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.44500000000000001,"15min":0.45000000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.54900000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.42199999999999999,"15min":0.42699999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.375}]},{"osd":4,"last update":"Thu Apr 25 04:17:14 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.75900000000000001,"15min":0.79000000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.76100000000000001,"5min":6.2409999999999997,"15min":6.2409999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.501}]},{"osd":5,"last update":"Thu Apr 25 04:16:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.433},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48899999999999999}]},{"osd":6,"last update":"Thu Apr 25 04:16:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.51000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:17:11 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.59099999999999997,"15min":0.60499999999999998},"min":{"1min":0.38100000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.61799999999999999,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.58599999999999997,"15min":0.59799999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.67300000000000004,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.36399999999999999}]},{"osd":1,"last update":"Thu Apr 25 04:17:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.58899999999999997,"15min":0.60299999999999998},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.70599999999999996,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.56100000000000005,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.63700000000000001,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.53000000000000003}]},{"osd":2,"last update":"Thu Apr 25 04:17:11 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.56999999999999995,"15min":0.58299999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.63900000000000001,"5min":1.081,"15min":1.081},"last":0.498},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.57099999999999995,"15min":0.57899999999999996},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.67800000000000005,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.50700000000000001}]},{"osd":3,"last update":"Thu Apr 25 04:17:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.59699999999999998,"15min":0.60899999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.75900000000000001,"5min":1.038,"15min":1.038},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.57299999999999995,"15min":0.58099999999999996},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.63300000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.38900000000000001}]},{"osd":5,"last update":"Thu Apr 25 04:16:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.58899999999999997}]},{"osd":6,"last update":"Thu Apr 25 04:17:00 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.57499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724566,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.46000000000000002}]},{"osd":1,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.503},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.53000000000000003}]},{"osd":2,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.63600000000000001}]},{"osd":3,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.56200000000000006}]},{"osd":4,"last update":"Thu Apr 25 04:16:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.65000000000000002}]},{"osd":6,"last update":"Thu Apr 25 04:16:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.58099999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-25T04:17:18.473 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-25T04:17:18.473 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-25T04:17:18.474 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-25T04:17:18.474 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-25T04:17:18.687 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/mon.smithi078/config 2024-04-25T04:17:19.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:18 smithi078 ceph-mon[21828]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:19.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:18 smithi078 ceph-mon[21828]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:18 smithi112 ceph-mon[24160]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:18 smithi112 ceph-mon[24160]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:20.445 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:17:20.445 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-25T04:17:21.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:20 smithi078 ceph-mon[21828]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:21.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:20 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1013492544' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-25T04:17:21.032 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-25T04:17:21.033 INFO:tasks.cephadm:Setup complete, yielding 2024-04-25T04:17:21.033 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-25T04:17:21.045 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:17:21.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch status' 2024-04-25T04:17:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:20 smithi112 ceph-mon[24160]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:20 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/1013492544' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-25T04:17:23.003 INFO:teuthology.orchestra.run.smithi078.stdout:Backend: cephadm 2024-04-25T04:17:23.004 INFO:teuthology.orchestra.run.smithi078.stdout:Available: Yes 2024-04-25T04:17:23.004 INFO:teuthology.orchestra.run.smithi078.stdout:Paused: No 2024-04-25T04:17:23.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:22 smithi078 ceph-mon[21828]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:23.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:22 smithi112 ceph-mon[24160]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:23.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-25T04:17:24.251 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:24 smithi078 ceph-mon[21828]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:24.251 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:24.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:24.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:24.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:24 smithi078 ceph-mon[21828]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:24.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:24 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:24 smithi112 ceph-mon[24160]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:24.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:24 smithi112 ceph-mon[24160]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:24.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:24 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:25.787 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:17:25.787 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c 243e09542a05 2024-04-25T04:17:25.787 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (5m) 65s ago 5m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 2m ago 5m 45.5M - 8.3.5 dad864ee21e9 9819914a14e8 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:9283 running (7m) 2m ago 7m 448M - 17.2.0 e1d6a67b021e 4dddbd296082 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (5m) 65s ago 5m 411M - 17.2.0 e1d6a67b021e 82627fc43e87 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 7m 44.5M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (4m) 65s ago 4m 39.9M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 2m ago 5m 15.4M - 1dbe0e931976 82c5e9a854e4 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (4m) 65s ago 4m 16.6M - 1dbe0e931976 92962d98d60e 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 4m 32.8M 3216M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 2m ago 3m 34.9M 3216M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 3m 31.0M 3216M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 2m 11.9M 3216M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (2m) 65s ago 2m 35.3M 3984M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (119s) 65s ago 118s 32.9M 3984M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:17:25.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (95s) 65s ago 95s 31.3M 3984M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:17:25.789 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (70s) 65s ago 70s 12.5M 3984M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:17:25.789 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 2m ago 4m 56.5M - 514e6a882f6e d366b695a24b 2024-04-25T04:17:26.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-25T04:17:26.962 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:26 smithi078 ceph-mon[21828]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:26.963 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:26 smithi078 ceph-mon[21828]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:27.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:26 smithi112 ceph-mon[24160]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:27.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:26 smithi112 ceph-mon[24160]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:28.305 INFO:teuthology.orchestra.run.smithi078.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-25T04:17:28.305 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-25T04:17:28.305 INFO:teuthology.orchestra.run.smithi078.stdout:crash 2/2 2m ago 6m * 2024-04-25T04:17:28.305 INFO:teuthology.orchestra.run.smithi078.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-25T04:17:28.306 INFO:teuthology.orchestra.run.smithi078.stdout:mgr 2/2 2m ago 6m count:2 2024-04-25T04:17:28.306 INFO:teuthology.orchestra.run.smithi078.stdout:mon 2/2 2m ago 5m smithi078:172.21.15.78=smithi078;smithi112:172.21.15.112=smithi112;count:2 2024-04-25T04:17:28.306 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-25T04:17:28.306 INFO:teuthology.orchestra.run.smithi078.stdout:osd 8 2m ago - 2024-04-25T04:17:28.306 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-25T04:17:29.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:28 smithi078 ceph-mon[21828]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:29.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-25T04:17:29.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:28 smithi112 ceph-mon[24160]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:30.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:29 smithi078 ceph-mon[21828]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:30.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:29 smithi112 ceph-mon[24160]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:31.028 INFO:teuthology.orchestra.run.smithi078.stdout:HOST ADDR LABELS STATUS 2024-04-25T04:17:31.028 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 172.21.15.78 2024-04-25T04:17:31.028 INFO:teuthology.orchestra.run.smithi078.stdout:smithi112 172.21.15.112 2024-04-25T04:17:31.028 INFO:teuthology.orchestra.run.smithi078.stdout:2 hosts in cluster 2024-04-25T04:17:31.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:30 smithi078 ceph-mon[21828]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:30 smithi112 ceph-mon[24160]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:31.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-25T04:17:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:32 smithi078 ceph-mon[21828]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:33.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:32 smithi078 ceph-mon[21828]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:33.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:32 smithi112 ceph-mon[24160]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:33.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:32 smithi112 ceph-mon[24160]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:33.492 INFO:teuthology.orchestra.run.smithi078.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-25T04:17:33.492 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400QR400BGN 400G LVM detected, locked 2024-04-25T04:17:33.492 INFO:teuthology.orchestra.run.smithi078.stdout:smithi112 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400NV400BGN 400G LVM detected, locked 2024-04-25T04:17:34.173 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-25T04:17:34.185 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:17:34.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-25T04:17:34.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:34 smithi078 ceph-mon[21828]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:34.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:34 smithi078 ceph-mon[21828]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:34.210 INFO:teuthology.orchestra.run.smithi078.stderr:+ systemctl stop nfs-server 2024-04-25T04:17:34.217 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi112.front.sepia.ceph.com 2024-04-25T04:17:34.217 DEBUG:teuthology.orchestra.run.smithi112:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-25T04:17:34.246 INFO:teuthology.orchestra.run.smithi112.stderr:+ systemctl stop nfs-server 2024-04-25T04:17:34.254 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-25T04:17:34.264 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:17:34.264 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-25T04:17:34.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:34 smithi112 ceph-mon[24160]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:34.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:34 smithi112 ceph-mon[24160]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:37.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:37 smithi078 ceph-mon[21828]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:37.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:37 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-25T04:17:37.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:37 smithi112 ceph-mon[24160]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:37.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:37 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-25T04:17:38.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:37 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[21824]: 2024-04-25T04:17:37.836+0000 7f8769db8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:17:38.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:38.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-25T04:17:38.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: osdmap e60: 8 total, 8 up, 8 in 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: pgmap v214: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: osdmap e61: 8 total, 8 up, 8 in 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: osdmap e62: 8 total, 8 up, 8 in 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: fsmap foofs:0 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: Saving service mds.foofs spec with placement count:2 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:38.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:38.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:38.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:38.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:38.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:17:38.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T04:17:38.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: osdmap e60: 8 total, 8 up, 8 in 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: pgmap v214: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: osdmap e61: 8 total, 8 up, 8 in 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: osdmap e62: 8 total, 8 up, 8 in 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-25T04:17:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: fsmap foofs:0 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: Saving service mds.foofs spec with placement count:2 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T04:17:38.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:38.627 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-25T04:17:38.638 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-25T04:17:38.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-25T04:17:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:39 smithi112 ceph-mon[24160]: Deploying daemon mds.foofs.smithi078.nlrtle on smithi078 2024-04-25T04:17:40.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:39 smithi112 ceph-mon[24160]: osdmap e63: 8 total, 8 up, 8 in 2024-04-25T04:17:40.185 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:39 smithi078 ceph-mon[21828]: Deploying daemon mds.foofs.smithi078.nlrtle on smithi078 2024-04-25T04:17:40.185 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:39 smithi078 ceph-mon[21828]: osdmap e63: 8 total, 8 up, 8 in 2024-04-25T04:17:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:40 smithi112 ceph-mon[24160]: pgmap v218: 65 pgs: 13 creating+peering, 23 unknown, 29 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:40 smithi112 ceph-mon[24160]: osdmap e64: 8 total, 8 up, 8 in 2024-04-25T04:17:41.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:40 smithi078 ceph-mon[21828]: pgmap v218: 65 pgs: 13 creating+peering, 23 unknown, 29 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:41.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:40 smithi078 ceph-mon[21828]: osdmap e64: 8 total, 8 up, 8 in 2024-04-25T04:17:41.710 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:17:41.711 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-25T04:10:57.785438Z", "last_refresh": "2024-04-25T04:14:49.672418Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-25T04:10:52.278017Z", "last_refresh": "2024-04-25T04:14:49.672485Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-25T04:10:55.081385Z", "last_refresh": "2024-04-25T04:14:49.672548Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-25T04:17:37.858013Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-25T04:17:37.851955Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-25T04:10:50.033060Z", "last_refresh": "2024-04-25T04:14:49.672348Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi112:172.21.15.112=smithi112"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-25T04:11:44.202951Z", "last_refresh": "2024-04-25T04:14:49.672209Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-25T04:10:56.481409Z", "last_refresh": "2024-04-25T04:14:49.672608Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-25T04:14:49.672729Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-25T04:10:53.692363Z", "last_refresh": "2024-04-25T04:14:49.672669Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-25T04:17:42.617 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-25T04:17:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:42 smithi112 ceph-mon[24160]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:42 smithi112 ceph-mon[24160]: pgmap v220: 65 pgs: 13 creating+peering, 16 unknown, 36 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:43.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:17:43.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T04:17:43.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:42 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:43.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:42 smithi078 ceph-mon[21828]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:43.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:42 smithi078 ceph-mon[21828]: pgmap v220: 65 pgs: 13 creating+peering, 16 unknown, 36 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:43.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:43.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:17:43.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-25T04:17:43.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:42 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:43.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: Deploying daemon mds.foofs.smithi112.xminls on smithi112 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: mds.? [v2:172.21.15.78:6834/131251842,v1:172.21.15.78:6835/131251842] up:boot 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: daemon mds.foofs.smithi078.nlrtle assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: Cluster is now healthy 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: fsmap foofs:0 1 up:standby 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:creating} 2024-04-25T04:17:43.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:43 smithi078 ceph-mon[21828]: daemon mds.foofs.smithi078.nlrtle is now active in filesystem foofs as rank 0 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: Deploying daemon mds.foofs.smithi112.xminls on smithi112 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: mds.? [v2:172.21.15.78:6834/131251842,v1:172.21.15.78:6835/131251842] up:boot 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: daemon mds.foofs.smithi078.nlrtle assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: Cluster is now healthy 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: fsmap foofs:0 1 up:standby 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:creating} 2024-04-25T04:17:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:43 smithi112 ceph-mon[24160]: daemon mds.foofs.smithi078.nlrtle is now active in filesystem foofs as rank 0 2024-04-25T04:17:44.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:44 smithi078 ceph-mon[21828]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:44.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:44 smithi078 ceph-mon[21828]: mds.? [v2:172.21.15.78:6834/131251842,v1:172.21.15.78:6835/131251842] up:active 2024-04-25T04:17:44.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:44 smithi078 ceph-mon[21828]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 2024-04-25T04:17:44.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:44 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:44.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:44 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:44.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:44 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:44.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:44 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:45.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:44 smithi112 ceph-mon[24160]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:17:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:44 smithi112 ceph-mon[24160]: mds.? [v2:172.21.15.78:6834/131251842,v1:172.21.15.78:6835/131251842] up:active 2024-04-25T04:17:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:44 smithi112 ceph-mon[24160]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 2024-04-25T04:17:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:44 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:44 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:44 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:44 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:45.693 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:17:45.693 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-25T04:10:57.785438Z", "last_refresh": "2024-04-25T04:14:49.672418Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-25T04:10:52.278017Z", "last_refresh": "2024-04-25T04:14:49.672485Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-25T04:10:55.081385Z", "last_refresh": "2024-04-25T04:14:49.672548Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-25T04:17:37.858013Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-25T04:17:37.851955Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-25T04:10:50.033060Z", "last_refresh": "2024-04-25T04:14:49.672348Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi112:172.21.15.112=smithi112"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-25T04:11:44.202951Z", "last_refresh": "2024-04-25T04:14:49.672209Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-25T04:10:56.481409Z", "last_refresh": "2024-04-25T04:14:49.672608Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-25T04:14:49.672729Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-25T04:10:53.692363Z", "last_refresh": "2024-04-25T04:14:49.672669Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-25T04:17:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:46 smithi112 ceph-mon[24160]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:boot 2024-04-25T04:17:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:46 smithi112 ceph-mon[24160]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:17:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:46 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:17:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:46 smithi112 ceph-mon[24160]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:17:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:46 smithi112 ceph-mon[24160]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:46.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:46 smithi112 ceph-mon[24160]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 387 B/s wr, 0 op/s 2024-04-25T04:17:46.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:46 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:46.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:46 smithi078 ceph-mon[21828]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:boot 2024-04-25T04:17:46.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:46 smithi078 ceph-mon[21828]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:17:46.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:46 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:17:46.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:46 smithi078 ceph-mon[21828]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:17:46.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:46 smithi078 ceph-mon[21828]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:46.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:46 smithi078 ceph-mon[21828]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 387 B/s wr, 0 op/s 2024-04-25T04:17:46.406 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:46 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:47.022 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-25T04:17:47.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:47 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:47 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:48.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-25T04:17:48.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:48 smithi078 ceph-mon[21828]: pgmap v223: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 687 B/s wr, 3 op/s 2024-04-25T04:17:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:48 smithi112 ceph-mon[24160]: pgmap v223: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 687 B/s wr, 3 op/s 2024-04-25T04:17:51.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:50 smithi078 ceph-mon[21828]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-25T04:17:51.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:50 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:51.146 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:17:51.146 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-25T04:10:57.785438Z", "last_refresh": "2024-04-25T04:14:49.672418Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-25T04:10:52.278017Z", "last_refresh": "2024-04-25T04:14:49.672485Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-25T04:10:55.081385Z", "last_refresh": "2024-04-25T04:14:49.672548Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-25T04:17:37.858013Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-25T04:17:37.851955Z", "last_refresh": "2024-04-25T04:17:46.586441Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-25T04:10:50.033060Z", "last_refresh": "2024-04-25T04:14:49.672348Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi112:172.21.15.112=smithi112"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-25T04:11:44.202951Z", "last_refresh": "2024-04-25T04:14:49.672209Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-25T04:10:56.481409Z", "last_refresh": "2024-04-25T04:14:49.672608Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-25T04:14:49.672729Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-25T04:10:53.692363Z", "last_refresh": "2024-04-25T04:14:49.672669Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-25T04:17:51.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:50 smithi112 ceph-mon[24160]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-04-25T04:17:51.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:50 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:52.414 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-25T04:17:53.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:53.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-25T04:17:53.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:53.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:53.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:53.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:53.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:53.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:52 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-25T04:17:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:17:53.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:17:53.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:17:53.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:52 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:17:53.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-25T04:17:54.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:54 smithi078 ceph-mon[21828]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-25T04:17:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:54 smithi112 ceph-mon[24160]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-25T04:17:55.387 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:17:55.388 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-25T04:10:57.785438Z", "last_refresh": "2024-04-25T04:17:52.650561Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-25T04:10:52.278017Z", "last_refresh": "2024-04-25T04:17:46.585831Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-25T04:10:55.081385Z", "last_refresh": "2024-04-25T04:17:52.650687Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-25T04:17:37.858013Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-25T04:17:37.851955Z", "last_refresh": "2024-04-25T04:17:46.586441Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-25T04:10:50.033060Z", "last_refresh": "2024-04-25T04:17:46.585994Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi112:172.21.15.112=smithi112"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-25T04:11:44.202951Z", "last_refresh": "2024-04-25T04:17:46.586065Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-25T04:10:56.481409Z", "last_refresh": "2024-04-25T04:17:46.586130Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-25T04:17:46.586207Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-25T04:10:53.692363Z", "last_refresh": "2024-04-25T04:17:52.650810Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-25T04:17:56.131 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-25T04:17:56.131 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-25T04:17:56.142 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:17:56.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-25T04:17:57.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:56 smithi078 ceph-mon[21828]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:57.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:56 smithi078 ceph-mon[21828]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-25T04:17:57.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:56 smithi112 ceph-mon[24160]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:17:57.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:56 smithi112 ceph-mon[24160]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-25T04:17:59.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:58 smithi078 ceph-mon[21828]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 3 op/s 2024-04-25T04:17:59.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-25T04:17:59.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:58 smithi112 ceph-mon[24160]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 3 op/s 2024-04-25T04:17:59.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-25T04:18:00.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:59 smithi078 ceph-mon[21828]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:00.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:59 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-25T04:18:00.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:59 smithi078 ceph-mon[21828]: osdmap e65: 8 total, 8 up, 8 in 2024-04-25T04:18:00.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:17:59 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-25T04:18:00.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:59 smithi112 ceph-mon[24160]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:00.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:59 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-25T04:18:00.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:59 smithi112 ceph-mon[24160]: osdmap e65: 8 total, 8 up, 8 in 2024-04-25T04:18:00.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:17:59 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-25T04:18:00.209 INFO:teuthology.orchestra.run.smithi078.stdout:NFS Cluster Created Successfully 2024-04-25T04:18:00.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-25T04:18:01.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: pgmap v230: 97 pgs: 21 creating+peering, 11 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:18:01.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-25T04:18:01.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: osdmap e66: 8 total, 8 up, 8 in 2024-04-25T04:18:01.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:01.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:18:01.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-25T04:18:01.107 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:00 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:01.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: pgmap v230: 97 pgs: 21 creating+peering, 11 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: osdmap e66: 8 total, 8 up, 8 in 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-25T04:18:01.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:18:01.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:18:01.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:01.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:18:01.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:18:01.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:18:01.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-25T04:18:01.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:00 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:02.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:01 smithi078 ceph-mon[21828]: Saving service nfs.foo spec with placement count:2 2024-04-25T04:18:02.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:01 smithi078 ceph-mon[21828]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx 2024-04-25T04:18:02.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:01 smithi078 ceph-mon[21828]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-25T04:18:02.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:01 smithi078 ceph-mon[21828]: Rados config object exists: conf-nfs.foo 2024-04-25T04:18:02.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:01 smithi078 ceph-mon[21828]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx-rgw 2024-04-25T04:18:02.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:01 smithi078 ceph-mon[21828]: Deploying daemon nfs.foo.0.0.smithi078.edyrsx on smithi078 2024-04-25T04:18:02.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:01 smithi078 ceph-mon[21828]: osdmap e67: 8 total, 8 up, 8 in 2024-04-25T04:18:02.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:01 smithi112 ceph-mon[24160]: Saving service nfs.foo spec with placement count:2 2024-04-25T04:18:02.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:01 smithi112 ceph-mon[24160]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx 2024-04-25T04:18:02.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:01 smithi112 ceph-mon[24160]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-25T04:18:02.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:01 smithi112 ceph-mon[24160]: Rados config object exists: conf-nfs.foo 2024-04-25T04:18:02.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:01 smithi112 ceph-mon[24160]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx-rgw 2024-04-25T04:18:02.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:01 smithi112 ceph-mon[24160]: Deploying daemon nfs.foo.0.0.smithi078.edyrsx on smithi078 2024-04-25T04:18:02.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:01 smithi112 ceph-mon[24160]: osdmap e67: 8 total, 8 up, 8 in 2024-04-25T04:18:02.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:02 smithi078 ceph-mon[21828]: pgmap v233: 97 pgs: 21 creating+peering, 8 unknown, 68 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:18:02.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:02 smithi078 ceph-mon[21828]: mgrmap e20: smithi078.eugwik(active, since 5m), standbys: smithi112.lvtlnw 2024-04-25T04:18:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:02 smithi112 ceph-mon[24160]: pgmap v233: 97 pgs: 21 creating+peering, 8 unknown, 68 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:18:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:02 smithi112 ceph-mon[24160]: mgrmap e20: smithi078.eugwik(active, since 5m), standbys: smithi112.lvtlnw 2024-04-25T04:18:03.952 INFO:teuthology.orchestra.run.smithi078.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-25T04:18:03.952 INFO:teuthology.orchestra.run.smithi078.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-25T04:18:03.952 INFO:teuthology.orchestra.run.smithi078.stderr:Error EINVAL: invalid command 2024-04-25T04:18:04.303 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:18:04.303 INFO:teuthology.orchestra.run.smithi078.stdout: "bind": "/fake", 2024-04-25T04:18:04.304 INFO:teuthology.orchestra.run.smithi078.stdout: "fs": "foofs", 2024-04-25T04:18:04.304 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-04-25T04:18:04.304 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster": "foo", 2024-04-25T04:18:04.304 INFO:teuthology.orchestra.run.smithi078.stdout: "mode": "RW" 2024-04-25T04:18:04.304 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:18:04.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:04 smithi078 ceph-mon[21828]: pgmap v234: 97 pgs: 21 creating+peering, 76 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:18:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:04 smithi112 ceph-mon[24160]: pgmap v234: 97 pgs: 21 creating+peering, 76 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-25T04:18:05.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:18:05.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:18:05.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:18:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:06.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:06 smithi078 ceph-mon[21828]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-25T04:18:06.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:18:06.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:18:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:06 smithi112 ceph-mon[24160]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-25T04:18:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:18:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:18:07.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:07 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:18:07.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:07 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:18:07.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:07 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:18:07.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:07 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:18:07.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:07 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:18:07.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:07 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:18:08.543 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:08 smithi112 ceph-mon[24160]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:08.544 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:08 smithi112 ceph-mon[24160]: Rados config object exists: conf-nfs.foo 2024-04-25T04:18:08.544 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:08 smithi112 ceph-mon[24160]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs-rgw 2024-04-25T04:18:08.544 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-25T04:18:08.544 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:08.544 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:08 smithi112 ceph-mon[24160]: Deploying daemon nfs.foo.1.0.smithi112.ouafxs on smithi112 2024-04-25T04:18:08.544 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:08 smithi112 ceph-mon[24160]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-25T04:18:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:08 smithi078 ceph-mon[21828]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:08 smithi078 ceph-mon[21828]: Rados config object exists: conf-nfs.foo 2024-04-25T04:18:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:08 smithi078 ceph-mon[21828]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs-rgw 2024-04-25T04:18:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-25T04:18:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:08 smithi078 ceph-mon[21828]: Deploying daemon nfs.foo.1.0.smithi112.ouafxs on smithi112 2024-04-25T04:18:08.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:08 smithi078 ceph-mon[21828]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-25T04:18:09.526 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:09 smithi112 ceph-mon[24160]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:09.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:09 smithi078 ceph-mon[21828]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:10.857 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:10.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:10.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:10.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:10 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:10.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:10 smithi078 ceph-mon[21828]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:10.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:10 smithi078 ceph-mon[21828]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-25T04:18:10.889 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:10.889 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:10.889 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:10.889 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:10 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:10.889 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:10 smithi112 ceph-mon[24160]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:10.890 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:10 smithi112 ceph-mon[24160]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-04-25T04:18:12.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:11 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:12.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:11 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:13.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:12 smithi078 ceph-mon[21828]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:13.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:12 smithi078 ceph-mon[21828]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 KiB/s wr, 7 op/s 2024-04-25T04:18:13.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:12 smithi112 ceph-mon[24160]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:12 smithi112 ceph-mon[24160]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 KiB/s wr, 7 op/s 2024-04-25T04:18:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:14.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:13 smithi078 ceph-mon[21828]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:14.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:13 smithi112 ceph-mon[24160]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:15.005 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:15.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:15.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:14 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:15.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-25T04:18:15.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:14 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:15.763 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-25T04:18:15.775 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:18:15.775 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-25T04:18:15.800 INFO:teuthology.orchestra.run.smithi078.stderr:+ mkdir /mnt/foo 2024-04-25T04:18:15.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-25T04:18:15.876 INFO:teuthology.orchestra.run.smithi078.stderr:++ hostname 2024-04-25T04:18:15.876 INFO:teuthology.orchestra.run.smithi078.stderr:+ mount -t nfs smithi078:/fake /mnt/foo -o sync 2024-04-25T04:18:16.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-25T04:18:16.317 INFO:teuthology.orchestra.run.smithi078.stderr:+ echo test 2024-04-25T04:18:16.344 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-25T04:18:16.406 INFO:teuthology.orchestra.run.smithi078.stderr:+ sync 2024-04-25T04:18:16.611 INFO:teuthology.run_tasks:Running task parallel... 2024-04-25T04:18:16.620 INFO:teuthology.task.parallel:starting parallel... 2024-04-25T04:18:16.621 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-25T04:18:16.621 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-25T04:18:16.621 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:18:16.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-25T04:18:16.622 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-25T04:18:16.622 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-25T04:18:16.623 INFO:teuthology.task.exec:Executing custom commands... 2024-04-25T04:18:16.623 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:18:16.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-25T04:18:16.685 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-25T04:18:16.685 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:18:16.685 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-25T04:18:16.685 INFO:teuthology.orchestra.run.smithi078.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-25T04:18:16.786 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-25T04:18:16.786 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-04-25T04:18:16.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:16 smithi078 ceph-mon[21828]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:16.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:16 smithi078 ceph-mon[21828]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-25T04:18:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:16 smithi112 ceph-mon[24160]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:16 smithi112 ceph-mon[24160]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-04-25T04:18:17.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112 25.95 MB/sec warmup 1 sec latency 39.021 ms 2024-04-25T04:18:18.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246 23.35 MB/sec warmup 2 sec latency 31.527 ms 2024-04-25T04:18:19.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:18 smithi078 ceph-mon[21828]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-25T04:18:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:18 smithi112 ceph-mon[24160]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-25T04:18:19.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 394 23.30 MB/sec warmup 3 sec latency 32.884 ms 2024-04-25T04:18:20.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-25T04:18:20.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 554 23.88 MB/sec warmup 4 sec latency 31.197 ms 2024-04-25T04:18:20.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:20 smithi078 ceph-mon[21828]: pgmap v242: 97 pgs: 97 active+clean; 39 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.3 MiB/s wr, 63 op/s 2024-04-25T04:18:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:20 smithi112 ceph-mon[24160]: pgmap v242: 97 pgs: 97 active+clean; 39 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.3 MiB/s wr, 63 op/s 2024-04-25T04:18:21.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 866 21.63 MB/sec warmup 5 sec latency 46.759 ms 2024-04-25T04:18:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 1368 20.72 MB/sec warmup 6 sec latency 61.200 ms 2024-04-25T04:18:22.917 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:22 smithi078 ceph-mon[21828]: pgmap v243: 97 pgs: 97 active+clean; 49 MiB data, 110 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 4.2 MiB/s wr, 199 op/s 2024-04-25T04:18:23.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:22 smithi112 ceph-mon[24160]: pgmap v243: 97 pgs: 97 active+clean; 49 MiB data, 110 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 4.2 MiB/s wr, 199 op/s 2024-04-25T04:18:23.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 2375 19.95 MB/sec warmup 7 sec latency 22.682 ms 2024-04-25T04:18:24.189 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:24 smithi078 ceph-mon[21828]: pgmap v244: 97 pgs: 97 active+clean; 92 MiB data, 151 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 8.1 MiB/s wr, 297 op/s 2024-04-25T04:18:24.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-25T04:18:24.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:24 smithi112 ceph-mon[24160]: pgmap v244: 97 pgs: 97 active+clean; 92 MiB data, 151 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 8.1 MiB/s wr, 297 op/s 2024-04-25T04:18:24.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3281 21.86 MB/sec warmup 8 sec latency 16.687 ms 2024-04-25T04:18:25.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3830 22.09 MB/sec warmup 9 sec latency 24.773 ms 2024-04-25T04:18:26.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 4437 20.36 MB/sec warmup 10 sec latency 22.155 ms 2024-04-25T04:18:26.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:26 smithi078 ceph-mon[21828]: pgmap v245: 97 pgs: 97 active+clean; 115 MiB data, 177 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 479 op/s 2024-04-25T04:18:27.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:26 smithi112 ceph-mon[24160]: pgmap v245: 97 pgs: 97 active+clean; 115 MiB data, 177 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 11 MiB/s wr, 479 op/s 2024-04-25T04:18:27.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 5194 20.59 MB/sec warmup 11 sec latency 99.884 ms 2024-04-25T04:18:28.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 5973 19.65 MB/sec warmup 12 sec latency 37.385 ms 2024-04-25T04:18:28.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:28 smithi078 ceph-mon[21828]: pgmap v246: 97 pgs: 97 active+clean; 116 MiB data, 177 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 11 MiB/s wr, 488 op/s 2024-04-25T04:18:29.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:28 smithi112 ceph-mon[24160]: pgmap v246: 97 pgs: 97 active+clean; 116 MiB data, 177 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 11 MiB/s wr, 488 op/s 2024-04-25T04:18:29.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph mgr module enable nfs --force' 2024-04-25T04:18:29.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 6752 20.60 MB/sec warmup 13 sec latency 29.039 ms 2024-04-25T04:18:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7361 21.03 MB/sec warmup 14 sec latency 23.979 ms 2024-04-25T04:18:31.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:30 smithi078 ceph-mon[21828]: pgmap v247: 97 pgs: 97 active+clean; 135 MiB data, 268 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 15 MiB/s wr, 896 op/s 2024-04-25T04:18:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:30 smithi112 ceph-mon[24160]: pgmap v247: 97 pgs: 97 active+clean; 135 MiB data, 268 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 15 MiB/s wr, 896 op/s 2024-04-25T04:18:31.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7854 19.85 MB/sec warmup 15 sec latency 33.456 ms 2024-04-25T04:18:32.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 8390 19.52 MB/sec warmup 16 sec latency 34.320 ms 2024-04-25T04:18:32.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:32 smithi078 ceph-mon[21828]: pgmap v248: 97 pgs: 97 active+clean; 139 MiB data, 293 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 12 MiB/s wr, 924 op/s 2024-04-25T04:18:33.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:32 smithi112 ceph-mon[24160]: pgmap v248: 97 pgs: 97 active+clean; 139 MiB data, 293 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 12 MiB/s wr, 924 op/s 2024-04-25T04:18:33.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9295 19.16 MB/sec warmup 17 sec latency 20.146 ms 2024-04-25T04:18:33.860 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:33 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3566058114' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-25T04:18:33.864 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:33 smithi078 ceph-mon[21828]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-25T04:18:33.890 INFO:teuthology.orchestra.run.smithi078.stderr:module 'nfs' is already enabled 2024-04-25T04:18:34.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:33 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/3566058114' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-25T04:18:34.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:33 smithi112 ceph-mon[24160]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-25T04:18:34.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9627 18.61 MB/sec warmup 18 sec latency 62.932 ms 2024-04-25T04:18:34.875 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:34 smithi078 ceph-mon[21828]: pgmap v249: 97 pgs: 97 active+clean; 145 MiB data, 344 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 13 MiB/s wr, 994 op/s 2024-04-25T04:18:34.875 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:34 smithi078 ceph-mon[21828]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-25T04:18:34.875 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:34 smithi078 ceph-mon[21828]: mgrmap e21: smithi078.eugwik(active, since 6m), standbys: smithi112.lvtlnw 2024-04-25T04:18:34.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-25T04:18:35.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:34 smithi112 ceph-mon[24160]: pgmap v249: 97 pgs: 97 active+clean; 145 MiB data, 344 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 13 MiB/s wr, 994 op/s 2024-04-25T04:18:35.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:34 smithi112 ceph-mon[24160]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-25T04:18:35.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:34 smithi112 ceph-mon[24160]: mgrmap e21: smithi078.eugwik(active, since 6m), standbys: smithi112.lvtlnw 2024-04-25T04:18:35.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10410 19.32 MB/sec warmup 19 sec latency 27.450 ms 2024-04-25T04:18:36.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11056 19.57 MB/sec warmup 20 sec latency 22.110 ms 2024-04-25T04:18:36.884 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:36 smithi078 ceph-mon[21828]: pgmap v250: 97 pgs: 97 active+clean; 141 MiB data, 359 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 11 MiB/s wr, 953 op/s 2024-04-25T04:18:37.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:36 smithi112 ceph-mon[24160]: pgmap v250: 97 pgs: 97 active+clean; 141 MiB data, 359 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 11 MiB/s wr, 953 op/s 2024-04-25T04:18:37.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11729 19.05 MB/sec warmup 21 sec latency 19.383 ms 2024-04-25T04:18:38.494 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 2024-04-25T04:18:38.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 12727 19.09 MB/sec warmup 22 sec latency 14.945 ms 2024-04-25T04:18:38.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:38 smithi078 ceph-mon[21828]: pgmap v251: 97 pgs: 97 active+clean; 142 MiB data, 369 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 783 op/s 2024-04-25T04:18:38.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:38.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:38.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:38.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:38 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:38 smithi112 ceph-mon[24160]: pgmap v251: 97 pgs: 97 active+clean; 142 MiB data, 369 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 783 op/s 2024-04-25T04:18:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:18:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:18:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:38 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:18:39.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 13557 19.63 MB/sec warmup 23 sec latency 20.076 ms 2024-04-25T04:18:39.807 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-25T04:18:39.808 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:18:39.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-25T04:18:39.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:39 smithi078 ceph-mon[21828]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:39.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:39 smithi078 ceph-mon[21828]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 2024-04-25T04:18:39.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:39 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:39.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:39 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:39 smithi112 ceph-mon[24160]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:39 smithi112 ceph-mon[24160]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 2024-04-25T04:18:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:39 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:39 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:18:40.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14367 20.10 MB/sec warmup 24 sec latency 32.710 ms 2024-04-25T04:18:40.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:40 smithi078 ceph-mon[21828]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 2024-04-25T04:18:40.905 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:40 smithi078 ceph-mon[21828]: pgmap v252: 97 pgs: 97 active+clean; 154 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2024-04-25T04:18:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:40 smithi112 ceph-mon[24160]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 2024-04-25T04:18:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:40 smithi112 ceph-mon[24160]: pgmap v252: 97 pgs: 97 active+clean; 154 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2024-04-25T04:18:41.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14816 19.51 MB/sec warmup 25 sec latency 28.905 ms 2024-04-25T04:18:42.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15534 19.38 MB/sec warmup 26 sec latency 20.946 ms 2024-04-25T04:18:42.916 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:42 smithi078 ceph-mon[21828]: pgmap v253: 97 pgs: 97 active+clean; 154 MiB data, 455 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 765 op/s 2024-04-25T04:18:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:42 smithi112 ceph-mon[24160]: pgmap v253: 97 pgs: 97 active+clean; 154 MiB data, 455 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 765 op/s 2024-04-25T04:18:43.720 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:18:43.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 16471 19.22 MB/sec warmup 27 sec latency 18.099 ms 2024-04-25T04:18:44.422 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:44 smithi078 ceph-mon[21828]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:44.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:44 smithi078 ceph-mon[21828]: pgmap v254: 97 pgs: 97 active+clean; 166 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-25T04:18:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:44 smithi112 ceph-mon[24160]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:44 smithi112 ceph-mon[24160]: pgmap v254: 97 pgs: 97 active+clean; 166 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-04-25T04:18:44.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 17199 19.65 MB/sec warmup 28 sec latency 22.351 ms 2024-04-25T04:18:44.814 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 30s ago 7m 22.1M - ba2b418f427c 243e09542a05 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 30s ago 7m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (6m) 32s ago 6m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 30s ago 6m 51.7M - 8.3.5 dad864ee21e9 9819914a14e8 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (63s) 30s ago 62s 15.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (60s) 32s ago 60s 12.1M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:9283 running (8m) 30s ago 8m 461M - 17.2.0 e1d6a67b021e 4dddbd296082 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (6m) 32s ago 6m 411M - 17.2.0 e1d6a67b021e 82627fc43e87 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 30s ago 8m 45.8M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (6m) 32s ago 6m 38.1M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:18:44.815 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (41s) 30s ago 40s 49.0M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (35s) 32s ago 35s 48.7M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 30s ago 6m 15.7M - 1dbe0e931976 82c5e9a854e4 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (6m) 32s ago 6m 16.8M - 1dbe0e931976 92962d98d60e 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 30s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 30s ago 4m 43.3M 3216M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 30s ago 4m 41.0M 3216M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 30s ago 4m 39.3M 3216M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (3m) 32s ago 3m 40.5M 3984M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (3m) 32s ago 3m 39.0M 3984M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (2m) 32s ago 2m 40.0M 3984M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (2m) 32s ago 2m 40.7M 3984M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:18:44.816 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 30s ago 5m 63.1M - 514e6a882f6e d366b695a24b 2024-04-25T04:18:45.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:45 smithi078 ceph-mon[21828]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:45.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:45 smithi078 ceph-mon[21828]: from='client.24369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:45.457 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:18:45.457 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:18:45.457 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:18:45.457 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:18:45.457 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:18:45.458 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:18:45.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:45 smithi112 ceph-mon[24160]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:45.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:45 smithi112 ceph-mon[24160]: from='client.24369 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:45.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 17923 19.94 MB/sec warmup 29 sec latency 25.347 ms 2024-04-25T04:18:46.012 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:18:46.012 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737", 2024-04-25T04:18:46.012 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:18:46.012 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-04-25T04:18:46.013 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "", 2024-04-25T04:18:46.013 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 image" 2024-04-25T04:18:46.013 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:18:46.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:46 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2089237243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:18:46.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:46 smithi078 ceph-mon[21828]: pgmap v255: 97 pgs: 97 active+clean; 168 MiB data, 557 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.9 MiB/s wr, 871 op/s 2024-04-25T04:18:46.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:46 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2089237243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:18:46.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:46 smithi112 ceph-mon[24160]: pgmap v255: 97 pgs: 97 active+clean; 168 MiB data, 557 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.9 MiB/s wr, 871 op/s 2024-04-25T04:18:46.651 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:18:46.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18360 19.45 MB/sec warmup 30 sec latency 18.180 ms 2024-04-25T04:18:47.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:47 smithi078 ceph-mon[21828]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:47.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:47 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/471274996' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:18:47.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:47 smithi112 ceph-mon[24160]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:18:47.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:47 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/471274996' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:18:47.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18983 19.26 MB/sec warmup 31 sec latency 25.209 ms 2024-04-25T04:18:48.205 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:48 smithi078 ceph-mon[21828]: pgmap v256: 97 pgs: 97 active+clean; 169 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.9 MiB/s wr, 820 op/s 2024-04-25T04:18:48.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:48 smithi112 ceph-mon[24160]: pgmap v256: 97 pgs: 97 active+clean; 169 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.9 MiB/s wr, 820 op/s 2024-04-25T04:18:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 19711 19.08 MB/sec warmup 32 sec latency 18.280 ms 2024-04-25T04:18:49.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 20307 18.89 MB/sec warmup 33 sec latency 39.816 ms 2024-04-25T04:18:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21252 19.47 MB/sec warmup 34 sec latency 38.243 ms 2024-04-25T04:18:50.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:50 smithi078 ceph-mon[21828]: pgmap v257: 97 pgs: 97 active+clean; 176 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-04-25T04:18:51.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:50 smithi112 ceph-mon[24160]: pgmap v257: 97 pgs: 97 active+clean; 176 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.17k op/s 2024-04-25T04:18:51.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21877 19.40 MB/sec warmup 35 sec latency 23.576 ms 2024-04-25T04:18:52.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 22503 19.19 MB/sec warmup 36 sec latency 20.989 ms 2024-04-25T04:18:52.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:52 smithi078 ceph-mon[21828]: pgmap v258: 97 pgs: 97 active+clean; 177 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 784 op/s 2024-04-25T04:18:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:52 smithi112 ceph-mon[24160]: pgmap v258: 97 pgs: 97 active+clean; 177 MiB data, 662 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 784 op/s 2024-04-25T04:18:53.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 23402 19.14 MB/sec warmup 37 sec latency 20.354 ms 2024-04-25T04:18:54.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:54 smithi078 ceph-mon[21828]: pgmap v259: 97 pgs: 97 active+clean; 184 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-25T04:18:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:54 smithi112 ceph-mon[24160]: pgmap v259: 97 pgs: 97 active+clean; 184 MiB data, 686 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-25T04:18:54.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 24469 19.60 MB/sec warmup 38 sec latency 17.320 ms 2024-04-25T04:18:55.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 25227 19.84 MB/sec warmup 39 sec latency 14.504 ms 2024-04-25T04:18:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 25921 19.54 MB/sec warmup 40 sec latency 23.659 ms 2024-04-25T04:18:56.966 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:56 smithi078 ceph-mon[21828]: pgmap v260: 97 pgs: 97 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 752 op/s 2024-04-25T04:18:57.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:56 smithi112 ceph-mon[24160]: pgmap v260: 97 pgs: 97 active+clean; 186 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 752 op/s 2024-04-25T04:18:57.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 26831 19.58 MB/sec warmup 41 sec latency 33.140 ms 2024-04-25T04:18:58.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 27547 19.56 MB/sec warmup 42 sec latency 24.923 ms 2024-04-25T04:18:59.006 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:18:58 smithi078 ceph-mon[21828]: pgmap v261: 97 pgs: 97 active+clean; 188 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.4 MiB/s wr, 771 op/s 2024-04-25T04:18:59.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:18:58 smithi112 ceph-mon[24160]: pgmap v261: 97 pgs: 97 active+clean; 188 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.4 MiB/s wr, 771 op/s 2024-04-25T04:18:59.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 28409 19.94 MB/sec warmup 43 sec latency 30.587 ms 2024-04-25T04:19:00.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 29133 19.81 MB/sec warmup 44 sec latency 18.342 ms 2024-04-25T04:19:01.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:00 smithi078 ceph-mon[21828]: pgmap v262: 97 pgs: 97 active+clean; 209 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.27k op/s 2024-04-25T04:19:01.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:00 smithi112 ceph-mon[24160]: pgmap v262: 97 pgs: 97 active+clean; 209 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 13 MiB/s wr, 1.27k op/s 2024-04-25T04:19:01.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 29995 19.84 MB/sec warmup 45 sec latency 13.749 ms 2024-04-25T04:19:02.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30930 19.78 MB/sec warmup 46 sec latency 21.151 ms 2024-04-25T04:19:03.025 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:02 smithi078 ceph-mon[21828]: pgmap v263: 97 pgs: 97 active+clean; 213 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-04-25T04:19:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:02 smithi112 ceph-mon[24160]: pgmap v263: 97 pgs: 97 active+clean; 213 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 995 op/s 2024-04-25T04:19:03.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 31872 20.18 MB/sec warmup 47 sec latency 25.881 ms 2024-04-25T04:19:04.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:04 smithi078 ceph-mon[21828]: pgmap v264: 97 pgs: 97 active+clean; 215 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.18k op/s 2024-04-25T04:19:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:04 smithi112 ceph-mon[24160]: pgmap v264: 97 pgs: 97 active+clean; 215 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 12 MiB/s wr, 1.18k op/s 2024-04-25T04:19:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32452 20.12 MB/sec warmup 48 sec latency 16.090 ms 2024-04-25T04:19:05.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 33084 19.92 MB/sec warmup 49 sec latency 14.565 ms 2024-04-25T04:19:06.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-04-25T04:19:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-25T04:19:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-04-25T04:19:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-25T04:19:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-04-25T04:19:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-25T04:19:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-04-25T04:19:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-04-25T04:19:06.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 33952 19.88 MB/sec warmup 50 sec latency 28.065 ms 2024-04-25T04:19:07.016 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-04-25T04:19:07.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-25T04:19:07.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-04-25T04:19:07.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-04-25T04:19:07.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: osdmap e68: 8 total, 8 up, 8 in 2024-04-25T04:19:07.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: pgmap v266: 97 pgs: 97 active+clean; 217 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2024-04-25T04:19:07.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:19:07.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:06 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:19:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-04-25T04:19:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-25T04:19:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-04-25T04:19:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-04-25T04:19:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: osdmap e68: 8 total, 8 up, 8 in 2024-04-25T04:19:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: pgmap v266: 97 pgs: 97 active+clean; 217 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2024-04-25T04:19:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:19:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:06 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:19:07.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 34641 19.84 MB/sec warmup 51 sec latency 22.908 ms 2024-04-25T04:19:08.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:07 smithi078 ceph-mon[21828]: osdmap e69: 8 total, 8 up, 8 in 2024-04-25T04:19:08.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:07 smithi112 ceph-mon[24160]: osdmap e69: 8 total, 8 up, 8 in 2024-04-25T04:19:08.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 35580 20.20 MB/sec warmup 52 sec latency 17.330 ms 2024-04-25T04:19:09.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:08 smithi078 ceph-mon[21828]: pgmap v268: 97 pgs: 97 active+clean; 220 MiB data, 927 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 7.5 MiB/s wr, 691 op/s 2024-04-25T04:19:09.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:08 smithi078 ceph-mon[21828]: osdmap e70: 8 total, 8 up, 8 in 2024-04-25T04:19:09.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:08 smithi078 ceph-mon[21828]: osdmap e71: 8 total, 8 up, 8 in 2024-04-25T04:19:09.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:08 smithi112 ceph-mon[24160]: pgmap v268: 97 pgs: 97 active+clean; 220 MiB data, 927 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 7.5 MiB/s wr, 691 op/s 2024-04-25T04:19:09.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:08 smithi112 ceph-mon[24160]: osdmap e70: 8 total, 8 up, 8 in 2024-04-25T04:19:09.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:08 smithi112 ceph-mon[24160]: osdmap e71: 8 total, 8 up, 8 in 2024-04-25T04:19:09.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 36267 20.07 MB/sec warmup 53 sec latency 21.098 ms 2024-04-25T04:19:10.282 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:10 smithi078 ceph-mon[21828]: osdmap e72: 8 total, 8 up, 8 in 2024-04-25T04:19:10.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:10 smithi078 ceph-mon[21828]: pgmap v272: 97 pgs: 1 activating+undersized+degraded+remapped, 96 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.21k op/s; 3/1566 objects degraded (0.192%) 2024-04-25T04:19:10.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:10 smithi112 ceph-mon[24160]: osdmap e72: 8 total, 8 up, 8 in 2024-04-25T04:19:10.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:10 smithi112 ceph-mon[24160]: pgmap v272: 97 pgs: 1 activating+undersized+degraded+remapped, 96 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 13 MiB/s wr, 1.21k op/s; 3/1566 objects degraded (0.192%) 2024-04-25T04:19:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 37018 20.02 MB/sec warmup 54 sec latency 21.705 ms 2024-04-25T04:19:11.288 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:11 smithi078 ceph-mon[21828]: Health check failed: Degraded data redundancy: 3/1566 objects degraded (0.192%), 1 pg degraded (PG_DEGRADED) 2024-04-25T04:19:11.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:11 smithi112 ceph-mon[24160]: Health check failed: Degraded data redundancy: 3/1566 objects degraded (0.192%), 1 pg degraded (PG_DEGRADED) 2024-04-25T04:19:11.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 37813 19.90 MB/sec warmup 55 sec latency 26.820 ms 2024-04-25T04:19:12.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:12 smithi078 ceph-mon[21828]: pgmap v273: 97 pgs: 1 activating+undersized+degraded+remapped, 96 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.00k op/s; 3/1593 objects degraded (0.188%) 2024-04-25T04:19:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:12 smithi112 ceph-mon[24160]: pgmap v273: 97 pgs: 1 activating+undersized+degraded+remapped, 96 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.00k op/s; 3/1593 objects degraded (0.188%) 2024-04-25T04:19:12.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 38420 19.94 MB/sec warmup 56 sec latency 29.664 ms 2024-04-25T04:19:13.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39121 20.10 MB/sec warmup 57 sec latency 28.516 ms 2024-04-25T04:19:14.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:14 smithi078 ceph-mon[21828]: pgmap v274: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.17k op/s; 682 KiB/s, 0 objects/s recovering 2024-04-25T04:19:14.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:14 smithi112 ceph-mon[24160]: pgmap v274: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.17k op/s; 682 KiB/s, 0 objects/s recovering 2024-04-25T04:19:14.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39776 19.99 MB/sec warmup 58 sec latency 13.394 ms 2024-04-25T04:19:15.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:15 smithi078 ceph-mon[21828]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/1593 objects degraded (0.188%), 1 pg degraded) 2024-04-25T04:19:15.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:15 smithi078 ceph-mon[21828]: Cluster is now healthy 2024-04-25T04:19:15.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:15 smithi112 ceph-mon[24160]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/1593 objects degraded (0.188%), 1 pg degraded) 2024-04-25T04:19:15.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:15 smithi112 ceph-mon[24160]: Cluster is now healthy 2024-04-25T04:19:15.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 40550 19.94 MB/sec warmup 59 sec latency 35.531 ms 2024-04-25T04:19:16.331 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:16 smithi078 ceph-mon[21828]: pgmap v275: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 12 MiB/s wr, 966 op/s; 513 KiB/s, 0 objects/s recovering 2024-04-25T04:19:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:16 smithi112 ceph-mon[24160]: pgmap v275: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 12 MiB/s wr, 966 op/s; 513 KiB/s, 0 objects/s recovering 2024-04-25T04:19:16.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41386 19.88 MB/sec warmup 60 sec latency 29.260 ms 2024-04-25T04:19:17.227 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:19:17.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41937 19.87 MB/sec warmup 61 sec latency 43.397 ms 2024-04-25T04:19:18.206 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:19:18.206 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 63s ago 7m 22.1M - ba2b418f427c 243e09542a05 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 63s ago 7m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (7m) 65s ago 6m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 63s ago 7m 51.7M - 8.3.5 dad864ee21e9 9819914a14e8 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (96s) 63s ago 96s 15.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (94s) 65s ago 93s 12.1M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:9283 running (8m) 63s ago 8m 461M - 17.2.0 e1d6a67b021e 4dddbd296082 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (6m) 65s ago 6m 411M - 17.2.0 e1d6a67b021e 82627fc43e87 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 63s ago 9m 45.8M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (6m) 65s ago 6m 38.1M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (74s) 63s ago 73s 49.0M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (69s) 65s ago 68s 48.7M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 63s ago 7m 15.7M - 1dbe0e931976 82c5e9a854e4 2024-04-25T04:19:18.207 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (6m) 65s ago 6m 16.8M - 1dbe0e931976 92962d98d60e 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 63s ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 63s ago 5m 43.3M 3216M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 63s ago 5m 41.0M 3216M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 63s ago 4m 39.3M 3216M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (4m) 65s ago 4m 40.5M 3984M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (3m) 65s ago 3m 39.0M 3984M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (3m) 65s ago 3m 40.0M 3984M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (3m) 65s ago 3m 40.7M 3984M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:19:18.208 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 63s ago 6m 63.1M - 514e6a882f6e d366b695a24b 2024-04-25T04:19:18.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 42601 19.99 MB/sec warmup 62 sec latency 49.348 ms 2024-04-25T04:19:18.851 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:18.852 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:19:18.853 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-25T04:19:18.853 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:19:18.853 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:19:19.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:18 smithi078 ceph-mon[21828]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:19.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:18 smithi078 ceph-mon[21828]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:19.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:18 smithi078 ceph-mon[21828]: pgmap v276: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 946 op/s; 426 KiB/s, 0 objects/s recovering 2024-04-25T04:19:19.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:18 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:19.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:18 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:19:19.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:18 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:19.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:18 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2313473857' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:19:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:18 smithi112 ceph-mon[24160]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:18 smithi112 ceph-mon[24160]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:18 smithi112 ceph-mon[24160]: pgmap v276: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 946 op/s; 426 KiB/s, 0 objects/s recovering 2024-04-25T04:19:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:18 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:19.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:18 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:19:19.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:18 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:19.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:18 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2313473857' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:19:19.352 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:19:19.352 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:19:19.352 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:19:19.352 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-04-25T04:19:19.352 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-04-25T04:19:19.353 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 image on host smithi112" 2024-04-25T04:19:19.353 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:19:19.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43054 19.87 MB/sec warmup 63 sec latency 23.052 ms 2024-04-25T04:19:19.937 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:19:20.111 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:19 smithi078 ceph-mon[21828]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:20.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:19 smithi078 ceph-mon[21828]: Upgrade: Target is version 19.0.0-3293-gd6f5f4e5 (unknown) 2024-04-25T04:19:20.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:19 smithi078 ceph-mon[21828]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72'] 2024-04-25T04:19:20.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:19 smithi078 ceph-mon[21828]: Upgrade: Need to upgrade myself (mgr.smithi078.eugwik) 2024-04-25T04:19:20.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:19 smithi078 ceph-mon[21828]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 on smithi112 2024-04-25T04:19:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:19 smithi112 ceph-mon[24160]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:19 smithi112 ceph-mon[24160]: Upgrade: Target is version 19.0.0-3293-gd6f5f4e5 (unknown) 2024-04-25T04:19:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:19 smithi112 ceph-mon[24160]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72'] 2024-04-25T04:19:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:19 smithi112 ceph-mon[24160]: Upgrade: Need to upgrade myself (mgr.smithi078.eugwik) 2024-04-25T04:19:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:19 smithi112 ceph-mon[24160]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 on smithi112 2024-04-25T04:19:20.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43668 19.72 MB/sec warmup 64 sec latency 21.290 ms 2024-04-25T04:19:21.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:20 smithi078 ceph-mon[21828]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:21.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:20 smithi078 ceph-mon[21828]: pgmap v277: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.02k op/s; 386 KiB/s, 0 objects/s recovering 2024-04-25T04:19:21.012 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:20 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3036370087' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:19:21.012 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:20 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:20 smithi112 ceph-mon[24160]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:20 smithi112 ceph-mon[24160]: pgmap v277: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.02k op/s; 386 KiB/s, 0 objects/s recovering 2024-04-25T04:19:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:20 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/3036370087' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:19:21.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:20 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:21.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 44444 19.70 MB/sec warmup 65 sec latency 27.357 ms 2024-04-25T04:19:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 45098 19.58 MB/sec warmup 66 sec latency 84.031 ms 2024-04-25T04:19:23.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:22 smithi078 ceph-mon[21828]: pgmap v278: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 826 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-25T04:19:23.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:22 smithi112 ceph-mon[24160]: pgmap v278: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 826 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-25T04:19:23.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 45524 19.56 MB/sec warmup 67 sec latency 39.212 ms 2024-04-25T04:19:24.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:24 smithi078 ceph-mon[21828]: pgmap v279: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 880 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-25T04:19:24.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:24 smithi112 ceph-mon[24160]: pgmap v279: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 880 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-25T04:19:24.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46033 19.59 MB/sec warmup 68 sec latency 53.918 ms 2024-04-25T04:19:25.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46491 19.50 MB/sec warmup 69 sec latency 37.076 ms 2024-04-25T04:19:26.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46963 19.30 MB/sec warmup 70 sec latency 23.290 ms 2024-04-25T04:19:27.121 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:26 smithi078 ceph-mon[21828]: pgmap v280: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 801 op/s 2024-04-25T04:19:27.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:26 smithi112 ceph-mon[24160]: pgmap v280: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 801 op/s 2024-04-25T04:19:27.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 47598 19.26 MB/sec warmup 71 sec latency 39.343 ms 2024-04-25T04:19:28.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48269 19.15 MB/sec warmup 72 sec latency 38.937 ms 2024-04-25T04:19:28.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:28 smithi078 ceph-mon[21828]: pgmap v281: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.1 MiB/s wr, 813 op/s 2024-04-25T04:19:29.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:28 smithi112 ceph-mon[24160]: pgmap v281: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.1 MiB/s wr, 813 op/s 2024-04-25T04:19:29.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48932 19.17 MB/sec warmup 73 sec latency 47.319 ms 2024-04-25T04:19:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 49684 19.31 MB/sec warmup 74 sec latency 23.301 ms 2024-04-25T04:19:30.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:30 smithi078 ceph-mon[21828]: pgmap v282: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 986 op/s 2024-04-25T04:19:31.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:30 smithi112 ceph-mon[24160]: pgmap v282: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 986 op/s 2024-04-25T04:19:31.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50097 19.23 MB/sec warmup 75 sec latency 51.310 ms 2024-04-25T04:19:32.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50519 19.08 MB/sec warmup 76 sec latency 33.530 ms 2024-04-25T04:19:32.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:32 smithi078 ceph-mon[21828]: pgmap v283: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 724 op/s 2024-04-25T04:19:33.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:32 smithi112 ceph-mon[24160]: pgmap v283: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 724 op/s 2024-04-25T04:19:33.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 51148 19.00 MB/sec warmup 77 sec latency 40.889 ms 2024-04-25T04:19:34.210 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:34 smithi078 ceph-mon[21828]: pgmap v284: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-04-25T04:19:34.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:34 smithi112 ceph-mon[24160]: pgmap v284: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.8 MiB/s wr, 871 op/s 2024-04-25T04:19:34.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 51841 18.96 MB/sec warmup 78 sec latency 28.598 ms 2024-04-25T04:19:35.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 52580 18.99 MB/sec warmup 79 sec latency 24.265 ms 2024-04-25T04:19:36.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53300 19.09 MB/sec warmup 80 sec latency 21.196 ms 2024-04-25T04:19:36.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:36 smithi078 ceph-mon[21828]: pgmap v285: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 822 op/s 2024-04-25T04:19:37.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:36 smithi112 ceph-mon[24160]: pgmap v285: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 822 op/s 2024-04-25T04:19:37.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53821 19.02 MB/sec warmup 81 sec latency 33.706 ms 2024-04-25T04:19:38.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 54321 18.89 MB/sec warmup 82 sec latency 44.133 ms 2024-04-25T04:19:38.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:38 smithi078 ceph-mon[21828]: pgmap v286: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 830 op/s 2024-04-25T04:19:39.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:38 smithi112 ceph-mon[24160]: pgmap v286: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 830 op/s 2024-04-25T04:19:39.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 54826 18.80 MB/sec warmup 83 sec latency 61.431 ms 2024-04-25T04:19:40.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 55578 18.78 MB/sec warmup 84 sec latency 22.140 ms 2024-04-25T04:19:41.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:40 smithi078 ceph-mon[21828]: pgmap v287: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-04-25T04:19:41.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:40 smithi112 ceph-mon[24160]: pgmap v287: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-04-25T04:19:41.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56251 18.81 MB/sec warmup 85 sec latency 25.027 ms 2024-04-25T04:19:42.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56884 18.87 MB/sec warmup 86 sec latency 24.943 ms 2024-04-25T04:19:42.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:42 smithi078 ceph-mon[21828]: pgmap v288: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 719 op/s 2024-04-25T04:19:43.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:42 smithi112 ceph-mon[24160]: pgmap v288: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 719 op/s 2024-04-25T04:19:43.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 57358 18.81 MB/sec warmup 87 sec latency 23.755 ms 2024-04-25T04:19:44.415 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:44 smithi078 ceph-mon[21828]: pgmap v289: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 808 op/s 2024-04-25T04:19:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:44 smithi112 ceph-mon[24160]: pgmap v289: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 808 op/s 2024-04-25T04:19:44.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 57806 18.68 MB/sec warmup 88 sec latency 28.263 ms 2024-04-25T04:19:45.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58338 18.59 MB/sec warmup 89 sec latency 28.384 ms 2024-04-25T04:19:46.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58926 18.57 MB/sec warmup 90 sec latency 27.822 ms 2024-04-25T04:19:46.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:46 smithi078 ceph-mon[21828]: pgmap v290: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 801 op/s 2024-04-25T04:19:47.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:46 smithi112 ceph-mon[24160]: pgmap v290: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 801 op/s 2024-04-25T04:19:47.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 59491 18.53 MB/sec warmup 91 sec latency 47.473 ms 2024-04-25T04:19:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 59892 18.46 MB/sec warmup 92 sec latency 34.340 ms 2024-04-25T04:19:49.013 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:48 smithi078 ceph-mon[21828]: pgmap v291: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-04-25T04:19:49.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:48 smithi112 ceph-mon[24160]: pgmap v291: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-04-25T04:19:49.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60470 18.49 MB/sec warmup 93 sec latency 29.185 ms 2024-04-25T04:19:50.424 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:19:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60943 18.43 MB/sec warmup 94 sec latency 29.446 ms 2024-04-25T04:19:51.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:50 smithi078 ceph-mon[21828]: pgmap v292: 97 pgs: 97 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 917 op/s 2024-04-25T04:19:51.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:50 smithi112 ceph-mon[24160]: pgmap v292: 97 pgs: 97 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 917 op/s 2024-04-25T04:19:51.417 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 97s ago 8m 22.1M - ba2b418f427c 243e09542a05 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 97s ago 8m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (7m) 99s ago 7m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 97s ago 7m 51.7M - 8.3.5 dad864ee21e9 9819914a14e8 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (2m) 97s ago 2m 15.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (2m) 99s ago 2m 12.1M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:9283 running (9m) 97s ago 9m 461M - 17.2.0 e1d6a67b021e 4dddbd296082 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (7m) 99s ago 7m 411M - 17.2.0 e1d6a67b021e 82627fc43e87 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 97s ago 9m 45.8M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (7m) 99s ago 7m 38.1M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (107s) 97s ago 107s 49.0M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (102s) 99s ago 101s 48.7M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:19:51.418 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 97s ago 7m 15.7M - 1dbe0e931976 82c5e9a854e4 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 99s ago 7m 16.8M - 1dbe0e931976 92962d98d60e 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 97s ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 97s ago 6m 43.3M 3216M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 97s ago 5m 41.0M 3216M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 97s ago 5m 39.3M 3216M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (4m) 99s ago 4m 40.5M 3984M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (4m) 99s ago 4m 39.0M 3984M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (4m) 99s ago 4m 40.0M 3984M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (3m) 99s ago 3m 40.7M 3984M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:19:51.419 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 97s ago 7m 63.1M - 514e6a882f6e d366b695a24b 2024-04-25T04:19:51.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61362 18.31 MB/sec warmup 95 sec latency 27.543 ms 2024-04-25T04:19:52.024 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:19:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:19:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:19:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:19:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:19:52.025 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:19:52.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:51 smithi078 ceph-mon[21828]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:52.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:51 smithi078 ceph-mon[21828]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:52.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:51 smithi112 ceph-mon[24160]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:52.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:51 smithi112 ceph-mon[24160]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:52.467 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:19:52.467 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:19:52.467 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:19:52.467 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-04-25T04:19:52.467 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-04-25T04:19:52.467 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 image on host smithi112" 2024-04-25T04:19:52.467 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:19:52.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61883 18.23 MB/sec warmup 96 sec latency 45.963 ms 2024-04-25T04:19:52.972 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:19:53.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:52 smithi078 ceph-mon[21828]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:53.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:52 smithi078 ceph-mon[21828]: pgmap v293: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 612 op/s 2024-04-25T04:19:53.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:52 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2897394137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:19:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:52 smithi112 ceph-mon[24160]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:53.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:52 smithi112 ceph-mon[24160]: pgmap v293: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 612 op/s 2024-04-25T04:19:53.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:52 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2897394137' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:19:53.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 62454 18.21 MB/sec warmup 97 sec latency 61.605 ms 2024-04-25T04:19:54.137 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:53 smithi078 ceph-mon[21828]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:54.137 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:53 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2525271527' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:19:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:53 smithi112 ceph-mon[24160]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:19:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:53 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2525271527' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:19:54.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 62967 18.12 MB/sec warmup 98 sec latency 34.322 ms 2024-04-25T04:19:55.017 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:54 smithi112 ceph-mon[24160]: pgmap v294: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 804 op/s 2024-04-25T04:19:55.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:54 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:55.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:54 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:19:55.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:54 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:19:55.018 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:54 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:19:55.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:54 smithi078 ceph-mon[21828]: pgmap v294: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 804 op/s 2024-04-25T04:19:55.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:54 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:55.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:54 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:19:55.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:54 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:19:55.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:54 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:19:55.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 63365 18.09 MB/sec warmup 99 sec latency 37.257 ms 2024-04-25T04:19:55.912 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:55 smithi078 ceph-mon[21828]: Upgrade: Updating mgr.smithi112.lvtlnw 2024-04-25T04:19:55.912 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:55 smithi078 ceph-mon[21828]: Deploying daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:19:56.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:55 smithi112 ceph-mon[24160]: Upgrade: Updating mgr.smithi112.lvtlnw 2024-04-25T04:19:56.242 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:55 smithi112 ceph-mon[24160]: Deploying daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:19:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 63829 18.12 MB/sec warmup 100 sec latency 35.311 ms 2024-04-25T04:19:56.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:56 smithi078 ceph-mon[21828]: pgmap v295: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.0 MiB/s wr, 825 op/s 2024-04-25T04:19:57.064 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:56 smithi112 ceph-mon[24160]: pgmap v295: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 8.0 MiB/s wr, 825 op/s 2024-04-25T04:19:57.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64297 18.06 MB/sec warmup 101 sec latency 108.905 ms 2024-04-25T04:19:58.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64693 17.97 MB/sec warmup 102 sec latency 32.282 ms 2024-04-25T04:19:58.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:58 smithi078 ceph-mon[21828]: pgmap v296: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.3 MiB/s wr, 650 op/s 2024-04-25T04:19:58.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:58.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:19:58.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:19:58.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:19:58 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:19:59.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:58 smithi112 ceph-mon[24160]: pgmap v296: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 6.3 MiB/s wr, 650 op/s 2024-04-25T04:19:59.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:19:59.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:19:59.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:19:59.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:19:58 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:19:59.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65053 17.83 MB/sec warmup 103 sec latency 118.083 ms 2024-04-25T04:20:00.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65453 17.74 MB/sec warmup 104 sec latency 68.592 ms 2024-04-25T04:20:00.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:00 smithi078 ceph-mon[21828]: pgmap v297: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 8.7 MiB/s wr, 890 op/s 2024-04-25T04:20:00.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:00 smithi078 ceph-mon[21828]: overall HEALTH_OK 2024-04-25T04:20:01.051 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:00 smithi112 ceph-mon[24160]: pgmap v297: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 8.7 MiB/s wr, 890 op/s 2024-04-25T04:20:01.051 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:00 smithi112 ceph-mon[24160]: overall HEALTH_OK 2024-04-25T04:20:01.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 66015 17.74 MB/sec warmup 105 sec latency 90.604 ms 2024-04-25T04:20:02.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 66680 17.74 MB/sec warmup 106 sec latency 23.905 ms 2024-04-25T04:20:03.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:02 smithi078 ceph-mon[21828]: pgmap v298: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 656 op/s 2024-04-25T04:20:03.206 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:02 smithi112 ceph-mon[24160]: pgmap v298: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 656 op/s 2024-04-25T04:20:03.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67162 17.75 MB/sec warmup 107 sec latency 36.765 ms 2024-04-25T04:20:04.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:04 smithi078 ceph-mon[21828]: pgmap v299: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 799 op/s 2024-04-25T04:20:04.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:04 smithi112 ceph-mon[24160]: pgmap v299: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 799 op/s 2024-04-25T04:20:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67641 17.71 MB/sec warmup 108 sec latency 85.831 ms 2024-04-25T04:20:05.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68080 17.65 MB/sec warmup 109 sec latency 36.122 ms 2024-04-25T04:20:06.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:20:06.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:05 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:20:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:20:06.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:05 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:20:06.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68467 17.55 MB/sec warmup 110 sec latency 52.108 ms 2024-04-25T04:20:07.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:06 smithi078 ceph-mon[21828]: pgmap v300: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.3 MiB/s wr, 794 op/s 2024-04-25T04:20:07.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:06 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:20:07.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:06 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:20:07.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:06 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:20:07.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:06 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:20:07.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:06 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:20:07.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:06 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:20:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:06 smithi112 ceph-mon[24160]: pgmap v300: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.3 MiB/s wr, 794 op/s 2024-04-25T04:20:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:06 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:20:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:06 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:20:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:06 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:20:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:06 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:20:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:06 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:20:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:06 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:20:07.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68953 17.48 MB/sec warmup 111 sec latency 43.663 ms 2024-04-25T04:20:08.125 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:07 smithi078 ceph-mon[21828]: mgrmap e22: smithi078.eugwik(active, since 8m), standbys: smithi112.lvtlnw 2024-04-25T04:20:08.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:07 smithi112 ceph-mon[24160]: mgrmap e22: smithi078.eugwik(active, since 8m), standbys: smithi112.lvtlnw 2024-04-25T04:20:08.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 69336 17.38 MB/sec warmup 112 sec latency 37.995 ms 2024-04-25T04:20:09.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:08 smithi078 ceph-mon[21828]: pgmap v301: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 717 op/s 2024-04-25T04:20:09.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:09.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:08 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:09.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:08 smithi112 ceph-mon[24160]: pgmap v301: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 717 op/s 2024-04-25T04:20:09.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:09.326 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:08 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:09.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 69748 17.35 MB/sec warmup 113 sec latency 36.756 ms 2024-04-25T04:20:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 70240 17.33 MB/sec warmup 114 sec latency 66.111 ms 2024-04-25T04:20:10.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:10 smithi078 ceph-mon[21828]: pgmap v302: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 8.1 MiB/s wr, 829 op/s 2024-04-25T04:20:11.231 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:10 smithi112 ceph-mon[24160]: pgmap v302: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 8.1 MiB/s wr, 829 op/s 2024-04-25T04:20:11.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 70724 17.35 MB/sec warmup 115 sec latency 44.114 ms 2024-04-25T04:20:12.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71192 17.31 MB/sec warmup 116 sec latency 32.794 ms 2024-04-25T04:20:13.501 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:12 smithi112 ceph-mon[24160]: pgmap v303: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.0 MiB/s wr, 664 op/s 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:12 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr fail", "who": "smithi078.eugwik"}]: dispatch 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:12 smithi112 ceph-mon[24160]: osdmap e73: 8 total, 8 up, 8 in 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:12 smithi078 ceph-mon[21828]: pgmap v303: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.0 MiB/s wr, 664 op/s 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:20:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:12 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr fail", "who": "smithi078.eugwik"}]: dispatch 2024-04-25T04:20:13.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:12 smithi078 ceph-mon[21828]: osdmap e73: 8 total, 8 up, 8 in 2024-04-25T04:20:13.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71697 17.29 MB/sec warmup 117 sec latency 23.183 ms 2024-04-25T04:20:13.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: Upgrade: Need to upgrade myself (mgr.smithi078.eugwik) 2024-04-25T04:20:13.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: Failing over to other MGR 2024-04-25T04:20:13.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:20:13.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:20:13.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "mgr fail", "who": "smithi078.eugwik"}]': finished 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: mgrmap e23: smithi112.lvtlnw(active, starting, since 0.994553s) 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:20:13.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: Manager daemon smithi112.lvtlnw is now available 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/trash_purge_schedule"}]: dispatch 2024-04-25T04:20:13.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:13 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/trash_purge_schedule"}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: Upgrade: Need to upgrade myself (mgr.smithi078.eugwik) 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: Failing over to other MGR 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:20:14.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.14204 172.21.15.78:0/1009024375' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "mgr fail", "who": "smithi078.eugwik"}]': finished 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: mgrmap e23: smithi112.lvtlnw(active, starting, since 0.994553s) 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: Manager daemon smithi112.lvtlnw is now available 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:20:14.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:20:14.403 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/trash_purge_schedule"}]: dispatch 2024-04-25T04:20:14.403 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:13 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi112.lvtlnw/trash_purge_schedule"}]: dispatch 2024-04-25T04:20:14.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72220 17.21 MB/sec warmup 118 sec latency 29.427 ms 2024-04-25T04:20:15.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:15 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:15.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:15 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:15.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:15 smithi078 ceph-mon[21828]: Deploying cephadm binary to smithi112 2024-04-25T04:20:15.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:15 smithi078 ceph-mon[21828]: Deploying cephadm binary to smithi078 2024-04-25T04:20:15.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:15 smithi078 ceph-mon[21828]: mgrmap e24: smithi112.lvtlnw(active, since 1.99791s) 2024-04-25T04:20:15.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:15 smithi078 ceph-mon[21828]: pgmap v3: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:15.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:15 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:15.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:15 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:15.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:15 smithi112 ceph-mon[24160]: Deploying cephadm binary to smithi112 2024-04-25T04:20:15.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:15 smithi112 ceph-mon[24160]: Deploying cephadm binary to smithi078 2024-04-25T04:20:15.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:15 smithi112 ceph-mon[24160]: mgrmap e24: smithi112.lvtlnw(active, since 1.99791s) 2024-04-25T04:20:15.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:15 smithi112 ceph-mon[24160]: pgmap v3: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:15.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72813 17.15 MB/sec warmup 119 sec latency 32.602 ms 2024-04-25T04:20:16.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:16 smithi078 ceph-mon[21828]: [25/Apr/2024:04:20:14] ENGINE Bus STARTING 2024-04-25T04:20:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:16 smithi078 ceph-mon[21828]: [25/Apr/2024:04:20:15] ENGINE Serving on http://172.21.15.112:8765 2024-04-25T04:20:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:16 smithi078 ceph-mon[21828]: [25/Apr/2024:04:20:15] ENGINE Serving on https://172.21.15.112:7150 2024-04-25T04:20:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:16 smithi078 ceph-mon[21828]: [25/Apr/2024:04:20:15] ENGINE Bus STARTED 2024-04-25T04:20:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:16 smithi078 ceph-mon[21828]: pgmap v4: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:16.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:16 smithi112 ceph-mon[24160]: [25/Apr/2024:04:20:14] ENGINE Bus STARTING 2024-04-25T04:20:16.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:16 smithi112 ceph-mon[24160]: [25/Apr/2024:04:20:15] ENGINE Serving on http://172.21.15.112:8765 2024-04-25T04:20:16.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:16 smithi112 ceph-mon[24160]: [25/Apr/2024:04:20:15] ENGINE Serving on https://172.21.15.112:7150 2024-04-25T04:20:16.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:16 smithi112 ceph-mon[24160]: [25/Apr/2024:04:20:15] ENGINE Bus STARTED 2024-04-25T04:20:16.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:16 smithi112 ceph-mon[24160]: pgmap v4: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:17.760 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:17 smithi078 ceph-mon[21828]: mgrmap e25: smithi112.lvtlnw(active, since 4s) 2024-04-25T04:20:17.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 73992 19.63 MB/sec execute 1 sec latency 32.070 ms 2024-04-25T04:20:17.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:17 smithi112 ceph-mon[24160]: mgrmap e25: smithi112.lvtlnw(active, since 4s) 2024-04-25T04:20:18.772 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:18 smithi078 ceph-mon[21828]: pgmap v5: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:18.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74449 19.42 MB/sec execute 2 sec latency 33.507 ms 2024-04-25T04:20:18.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:18 smithi112 ceph-mon[24160]: pgmap v5: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:19.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74882 16.51 MB/sec execute 3 sec latency 38.737 ms 2024-04-25T04:20:20.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:20 smithi078 ceph-mon[21828]: pgmap v6: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:20.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75418 15.57 MB/sec execute 4 sec latency 17.356 ms 2024-04-25T04:20:20.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:20 smithi112 ceph-mon[24160]: pgmap v6: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:20:21.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75898 14.39 MB/sec execute 5 sec latency 43.965 ms 2024-04-25T04:20:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76297 12.97 MB/sec execute 6 sec latency 64.676 ms 2024-04-25T04:20:22.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:22 smithi078 ceph-mon[21828]: pgmap v7: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-04-25T04:20:22.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:22 smithi078 ceph-mon[21828]: Standby manager daemon smithi078.eugwik started 2024-04-25T04:20:22.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:22 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/crt"}]: dispatch 2024-04-25T04:20:22.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:22 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:20:22.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:22 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/key"}]: dispatch 2024-04-25T04:20:22.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:22 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:20:22.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:22 smithi112 ceph-mon[24160]: pgmap v7: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-04-25T04:20:22.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:22 smithi112 ceph-mon[24160]: Standby manager daemon smithi078.eugwik started 2024-04-25T04:20:22.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:22 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/crt"}]: dispatch 2024-04-25T04:20:22.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:22 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:20:22.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:22 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/key"}]: dispatch 2024-04-25T04:20:22.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:22 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4177435336' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:20:23.552 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:20:23.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76756 13.19 MB/sec execute 7 sec latency 34.743 ms 2024-04-25T04:20:23.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:23 smithi078 ceph-mon[21828]: mgrmap e26: smithi112.lvtlnw(active, since 10s), standbys: smithi078.eugwik 2024-04-25T04:20:23.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:23 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:20:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:23 smithi112 ceph-mon[24160]: mgrmap e26: smithi112.lvtlnw(active, since 10s), standbys: smithi078.eugwik 2024-04-25T04:20:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:23 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 2m ago 8m 22.1M - ba2b418f427c 243e09542a05 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 2m ago 8m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (8m) 16s ago 8m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 2m ago 8m 51.7M - 8.3.5 dad864ee21e9 9819914a14e8 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (2m) 2m ago 2m 15.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (2m) 16s ago 2m 12.6M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:9283 running (10m) 2m ago 10m 461M - 17.2.0 e1d6a67b021e 4dddbd296082 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (27s) 16s ago 8m 74.5M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 676f050e5409 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 2m ago 10m 45.8M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:20:24.628 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (7m) 16s ago 7m 39.0M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (2m) 2m ago 2m 49.0M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (2m) 16s ago 2m 73.0M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 8m 15.7M - 1dbe0e931976 82c5e9a854e4 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 16s ago 7m 18.8M - 1dbe0e931976 92962d98d60e 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 2m ago 7m 43.0M 3216M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 2m ago 6m 43.3M 3216M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 2m ago 6m 41.0M 3216M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 2m ago 5m 39.3M 3216M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (5m) 16s ago 5m 233M 3984M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (4m) 16s ago 4m 283M 3984M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (4m) 16s ago 4m 252M 3984M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (4m) 16s ago 4m 343M 3984M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:20:24.629 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 7m 63.1M - 514e6a882f6e d366b695a24b 2024-04-25T04:20:24.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77435 13.91 MB/sec execute 8 sec latency 40.536 ms 2024-04-25T04:20:24.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:24 smithi078 ceph-mon[21828]: from='client.24453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:24.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:24 smithi078 ceph-mon[21828]: pgmap v8: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.5 MiB/s wr, 433 op/s 2024-04-25T04:20:24.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:24 smithi112 ceph-mon[24160]: from='client.24453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:24.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:24 smithi112 ceph-mon[24160]: pgmap v8: 97 pgs: 97 active+clean; 320 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.5 MiB/s wr, 433 op/s 2024-04-25T04:20:25.292 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:20:25.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:20:25.292 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:20:25.292 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:20:25.293 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:20:25.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:25 smithi078 ceph-mon[21828]: from='client.24457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:25.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:25 smithi078 ceph-mon[21828]: from='client.24461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:25.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:25 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/530519296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:20:25.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78045 15.06 MB/sec execute 9 sec latency 61.293 ms 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:20:25.805 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:20:25.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:25 smithi112 ceph-mon[24160]: from='client.24457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:25.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:25 smithi112 ceph-mon[24160]: from='client.24461 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:25.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:25 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/530519296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:20:26.482 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:20:26.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:26 smithi078 ceph-mon[21828]: pgmap v9: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.6 MiB/s wr, 678 op/s 2024-04-25T04:20:26.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:26 smithi078 ceph-mon[21828]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:26.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:26 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3857679127' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:20:26.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78771 15.56 MB/sec execute 10 sec latency 37.334 ms 2024-04-25T04:20:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:26 smithi112 ceph-mon[24160]: pgmap v9: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.6 MiB/s wr, 678 op/s 2024-04-25T04:20:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:26 smithi112 ceph-mon[24160]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:26.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:26 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/3857679127' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:20:27.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79454 15.30 MB/sec execute 11 sec latency 25.765 ms 2024-04-25T04:20:28.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:28 smithi078 ceph-mon[21828]: pgmap v10: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 620 op/s 2024-04-25T04:20:28.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80031 14.89 MB/sec execute 12 sec latency 33.957 ms 2024-04-25T04:20:28.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:28 smithi112 ceph-mon[24160]: pgmap v10: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 620 op/s 2024-04-25T04:20:29.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:29 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:29.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:29 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:29.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:29 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:29.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:29 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:29.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:29 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:29.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80536 14.98 MB/sec execute 13 sec latency 34.212 ms 2024-04-25T04:20:29.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:29 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:29.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:29 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:29.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:29 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:29.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:29 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:29.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:29 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:30.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:30 smithi078 ceph-mon[21828]: pgmap v11: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 620 op/s 2024-04-25T04:20:30.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:30 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:30.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:30 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81080 15.04 MB/sec execute 14 sec latency 36.790 ms 2024-04-25T04:20:30.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:30 smithi112 ceph-mon[24160]: pgmap v11: 97 pgs: 97 active+clean; 314 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.0 MiB/s wr, 620 op/s 2024-04-25T04:20:30.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:30 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:30.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:30 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:31.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81490 15.09 MB/sec execute 15 sec latency 30.331 ms 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi078 to 1936M 2024-04-25T04:20:32.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:32.046 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:32 smithi078 ceph-mon[21828]: pgmap v12: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 10 MiB/s wr, 992 op/s 2024-04-25T04:20:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi078 to 1936M 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:32 smithi112 ceph-mon[24160]: pgmap v12: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 10 MiB/s wr, 992 op/s 2024-04-25T04:20:32.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81965 15.06 MB/sec execute 16 sec latency 29.952 ms 2024-04-25T04:20:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82420 14.84 MB/sec execute 17 sec latency 53.040 ms 2024-04-25T04:20:34.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:34 smithi078 ceph-mon[21828]: pgmap v13: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.6 MiB/s wr, 668 op/s 2024-04-25T04:20:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:34 smithi112 ceph-mon[24160]: pgmap v13: 97 pgs: 97 active+clean; 317 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.6 MiB/s wr, 668 op/s 2024-04-25T04:20:34.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82877 14.46 MB/sec execute 18 sec latency 38.471 ms 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: Adjusting osd_memory_target on smithi112 to 2704M 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:20:35.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:35 smithi078 ceph-mon[21828]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:20:35.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83326 14.05 MB/sec execute 19 sec latency 33.370 ms 2024-04-25T04:20:35.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:35.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:35.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-25T04:20:35.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: Adjusting osd_memory_target on smithi112 to 2704M 2024-04-25T04:20:35.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:35.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:20:35.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:20:35.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:20:35.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:20:35.903 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:35 smithi112 ceph-mon[24160]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:20:36.793 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83788 14.07 MB/sec execute 20 sec latency 26.823 ms 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: pgmap v14: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 945 op/s 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:36 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:20:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:20:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: pgmap v14: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 945 op/s 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:36.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:36 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:37.793 INFO:teuthology.orchestra.run.smithi078.stdout: 5 84579 14.54 MB/sec execute 21 sec latency 28.383 ms 2024-04-25T04:20:37.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:37 smithi078 ceph-mon[21828]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-04-25T04:20:37.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:37 smithi078 ceph-mon[21828]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-04-25T04:20:37.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:37 smithi112 ceph-mon[24160]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-04-25T04:20:37.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:37 smithi112 ceph-mon[24160]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-04-25T04:20:38.793 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85226 14.88 MB/sec execute 22 sec latency 29.098 ms 2024-04-25T04:20:38.860 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:38 smithi078 ceph-mon[21828]: pgmap v15: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.0 MiB/s wr, 649 op/s 2024-04-25T04:20:38.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:38 smithi112 ceph-mon[24160]: pgmap v15: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.0 MiB/s wr, 649 op/s 2024-04-25T04:20:39.793 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85866 15.05 MB/sec execute 23 sec latency 28.222 ms 2024-04-25T04:20:40.793 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86510 14.97 MB/sec execute 24 sec latency 24.310 ms 2024-04-25T04:20:40.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:40 smithi078 ceph-mon[21828]: pgmap v16: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.0 MiB/s wr, 649 op/s 2024-04-25T04:20:40.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:40 smithi112 ceph-mon[24160]: pgmap v16: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.0 MiB/s wr, 649 op/s 2024-04-25T04:20:41.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87279 15.06 MB/sec execute 25 sec latency 25.973 ms 2024-04-25T04:20:42.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88173 15.57 MB/sec execute 26 sec latency 20.774 ms 2024-04-25T04:20:42.886 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:42 smithi078 ceph-mon[21828]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-04-25T04:20:42.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:42 smithi112 ceph-mon[24160]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 996 op/s 2024-04-25T04:20:43.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88813 15.69 MB/sec execute 27 sec latency 79.437 ms 2024-04-25T04:20:44.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:44 smithi078 ceph-mon[21828]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.7 MiB/s wr, 624 op/s 2024-04-25T04:20:44.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:44 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:44 smithi112 ceph-mon[24160]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.7 MiB/s wr, 624 op/s 2024-04-25T04:20:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:44 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:44.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89353 15.66 MB/sec execute 28 sec latency 21.162 ms 2024-04-25T04:20:45.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89863 15.52 MB/sec execute 29 sec latency 25.043 ms 2024-04-25T04:20:46.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:46 smithi078 ceph-mon[21828]: pgmap v19: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-25T04:20:46.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:46 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:46 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:46 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:46 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:46 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-25T04:20:46.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90353 15.24 MB/sec execute 30 sec latency 47.279 ms 2024-04-25T04:20:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:46 smithi112 ceph-mon[24160]: pgmap v19: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-25T04:20:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:46 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:46 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:46 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:46 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:46 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-25T04:20:47.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:47 smithi078 ceph-mon[21828]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-04-25T04:20:47.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:47 smithi078 ceph-mon[21828]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-25T04:20:47.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:47 smithi078 ceph-mon[21828]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-25T04:20:47.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:47 smithi078 ceph-mon[21828]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-04-25T04:20:47.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90787 15.19 MB/sec execute 31 sec latency 40.347 ms 2024-04-25T04:20:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:47 smithi112 ceph-mon[24160]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-04-25T04:20:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:47 smithi112 ceph-mon[24160]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-25T04:20:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:47 smithi112 ceph-mon[24160]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-25T04:20:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:47 smithi112 ceph-mon[24160]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-04-25T04:20:48.679 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:48 smithi078 ceph-mon[21828]: pgmap v20: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.9 MiB/s wr, 745 op/s 2024-04-25T04:20:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91344 15.32 MB/sec execute 32 sec latency 27.255 ms 2024-04-25T04:20:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:48 smithi112 ceph-mon[24160]: pgmap v20: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.9 MiB/s wr, 745 op/s 2024-04-25T04:20:49.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91904 15.31 MB/sec execute 33 sec latency 33.818 ms 2024-04-25T04:20:50.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:50 smithi078 ceph-mon[21828]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.9 MiB/s wr, 745 op/s 2024-04-25T04:20:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92391 15.31 MB/sec execute 34 sec latency 24.033 ms 2024-04-25T04:20:51.088 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:50 smithi112 ceph-mon[24160]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.9 MiB/s wr, 745 op/s 2024-04-25T04:20:51.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92891 15.19 MB/sec execute 35 sec latency 43.533 ms 2024-04-25T04:20:52.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:52 smithi078 ceph-mon[21828]: pgmap v22: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-25T04:20:52.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93415 15.16 MB/sec execute 36 sec latency 43.155 ms 2024-04-25T04:20:53.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:52 smithi112 ceph-mon[24160]: pgmap v22: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-25T04:20:53.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93992 15.01 MB/sec execute 37 sec latency 30.600 ms 2024-04-25T04:20:54.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:54 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:54.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:54 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:54.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:54 smithi078 ceph-mon[21828]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-04-25T04:20:54.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:54 smithi078 ceph-mon[21828]: Reconfiguring daemon node-exporter.smithi078 on smithi078 2024-04-25T04:20:54.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:54 smithi078 ceph-mon[21828]: pgmap v23: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 695 op/s 2024-04-25T04:20:54.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94457 14.96 MB/sec execute 38 sec latency 43.982 ms 2024-04-25T04:20:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:54 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:54 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:54 smithi112 ceph-mon[24160]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-04-25T04:20:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:54 smithi112 ceph-mon[24160]: Reconfiguring daemon node-exporter.smithi078 on smithi078 2024-04-25T04:20:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:54 smithi112 ceph-mon[24160]: pgmap v23: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 695 op/s 2024-04-25T04:20:55.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95011 15.09 MB/sec execute 39 sec latency 40.885 ms 2024-04-25T04:20:56.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:56 smithi078 ceph-mon[21828]: pgmap v24: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-25T04:20:56.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:56 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:56.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:56 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95405 14.94 MB/sec execute 40 sec latency 46.089 ms 2024-04-25T04:20:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:56 smithi112 ceph-mon[24160]: pgmap v24: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-25T04:20:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:56 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:56 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:20:56.991 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:20:57.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95716 14.86 MB/sec execute 41 sec latency 65.768 ms 2024-04-25T04:20:57.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:57 smithi078 ceph-mon[21828]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-04-25T04:20:57.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:57 smithi078 ceph-mon[21828]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-04-25T04:20:57.893 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:20:57.893 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 starting - - - - 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 28s ago 9m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (8m) 28s ago 8m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (3m) 28s ago 3m 38.8M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (3m) 28s ago 3m 13.0M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:9283 running (10m) 28s ago 10m 375M - 17.2.0 e1d6a67b021e 4dddbd296082 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (60s) 28s ago 8m 489M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 676f050e5409 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 28s ago 10m 51.2M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (8m) 28s ago 8m 40.1M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (2m) 28s ago 2m 273M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (2m) 28s ago 2m 75.4M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 starting - - - - 2024-04-25T04:20:57.894 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (8m) 28s ago 8m 18.0M - 1.3.1 1dbe0e931976 92962d98d60e 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 28s ago 7m 297M 1936M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 28s ago 7m 236M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 28s ago 6m 199M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 28s ago 6m 264M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (5m) 28s ago 5m 272M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (5m) 28s ago 5m 321M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (5m) 28s ago 5m 273M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (4m) 28s ago 4m 391M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:20:57.895 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 28s ago 8m 67.4M - 2.33.4 514e6a882f6e d366b695a24b 2024-04-25T04:20:57.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:57 smithi112 ceph-mon[24160]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-04-25T04:20:57.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:57 smithi112 ceph-mon[24160]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-04-25T04:20:58.465 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:20:58.465 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:20:58.466 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:20:58.466 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:58.466 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:20:58.466 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-25T04:20:58.466 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:20:58.466 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:58.467 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:20:58.467 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:20:58.467 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:58.467 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:20:58.467 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:20:58.467 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:20:58.468 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:20:58.468 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-25T04:20:58.468 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:20:58.468 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:20:58.468 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:20:58.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96081 14.78 MB/sec execute 42 sec latency 54.554 ms 2024-04-25T04:20:58.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:58 smithi078 ceph-mon[21828]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:58.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:58 smithi078 ceph-mon[21828]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:58.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:58 smithi078 ceph-mon[21828]: pgmap v25: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-04-25T04:20:58.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:58 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/1596012885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:20:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:58 smithi112 ceph-mon[24160]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:58 smithi112 ceph-mon[24160]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:58.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:58 smithi112 ceph-mon[24160]: pgmap v25: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-04-25T04:20:58.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:58 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/1596012885' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:20:58.942 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:20:58.942 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:20:58.942 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:20:58.943 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:20:58.943 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-04-25T04:20:58.943 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-04-25T04:20:58.943 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-04-25T04:20:58.943 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:20:58.943 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:20:59.517 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:20:59.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96435 14.58 MB/sec execute 43 sec latency 37.971 ms 2024-04-25T04:20:59.891 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:59 smithi078 ceph-mon[21828]: from='client.24481 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:59.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:59 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:59.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:20:59 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/2107352728' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:20:59.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:59 smithi112 ceph-mon[24160]: from='client.24481 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:20:59.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:59 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:20:59.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:20:59 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/2107352728' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:21:00.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96760 14.48 MB/sec execute 44 sec latency 46.105 ms 2024-04-25T04:21:00.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:00 smithi112 ceph-mon[24160]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:00.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:00 smithi112 ceph-mon[24160]: pgmap v26: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-04-25T04:21:00.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:00 smithi078 ceph-mon[21828]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:00.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:00 smithi078 ceph-mon[21828]: pgmap v26: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-04-25T04:21:01.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97272 14.38 MB/sec execute 45 sec latency 39.918 ms 2024-04-25T04:21:02.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:02 smithi078 ceph-mon[21828]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 853 op/s 2024-04-25T04:21:02.794 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97758 14.26 MB/sec execute 46 sec latency 46.067 ms 2024-04-25T04:21:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:02 smithi112 ceph-mon[24160]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 853 op/s 2024-04-25T04:21:03.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98249 14.28 MB/sec execute 47 sec latency 26.051 ms 2024-04-25T04:21:04.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:04 smithi078 ceph-mon[21828]: pgmap v28: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 557 op/s 2024-04-25T04:21:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:04 smithi112 ceph-mon[24160]: pgmap v28: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 557 op/s 2024-04-25T04:21:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98782 14.40 MB/sec execute 48 sec latency 60.041 ms 2024-04-25T04:21:05.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:05 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:05.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:05 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:05.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:05 smithi078 ceph-mon[21828]: Reconfiguring node-exporter.smithi112 (dependencies changed)... 2024-04-25T04:21:05.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:05 smithi078 ceph-mon[21828]: Reconfiguring daemon node-exporter.smithi112 on smithi112 2024-04-25T04:21:05.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99233 14.39 MB/sec execute 49 sec latency 32.635 ms 2024-04-25T04:21:05.802 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:05 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:05.802 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:05 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:05.802 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:05 smithi112 ceph-mon[24160]: Reconfiguring node-exporter.smithi112 (dependencies changed)... 2024-04-25T04:21:05.802 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:05 smithi112 ceph-mon[24160]: Reconfiguring daemon node-exporter.smithi112 on smithi112 2024-04-25T04:21:06.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:06 smithi078 ceph-mon[21828]: pgmap v29: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 838 op/s 2024-04-25T04:21:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:06 smithi112 ceph-mon[24160]: pgmap v29: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 838 op/s 2024-04-25T04:21:06.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99642 14.33 MB/sec execute 50 sec latency 40.981 ms 2024-04-25T04:21:07.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99956 14.17 MB/sec execute 51 sec latency 55.864 ms 2024-04-25T04:21:08.025 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.025 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.025 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:21:08.025 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-04-25T04:21:08.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-04-25T04:21:08.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:08.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:08.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:08.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:07 smithi078 ceph-mon[21828]: pgmap v30: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-04-25T04:21:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:08.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:07 smithi112 ceph-mon[24160]: pgmap v30: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-04-25T04:21:08.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100371 14.15 MB/sec execute 52 sec latency 40.149 ms 2024-04-25T04:21:09.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100811 14.01 MB/sec execute 53 sec latency 37.039 ms 2024-04-25T04:21:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101163 13.84 MB/sec execute 54 sec latency 43.733 ms 2024-04-25T04:21:10.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:10 smithi078 ceph-mon[21828]: pgmap v31: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-04-25T04:21:10.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:10 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:10.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:10 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:21:10.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:10 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:21:10.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:10 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:21:10.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:10 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:21:10.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:10 smithi112 ceph-mon[24160]: pgmap v31: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 520 op/s 2024-04-25T04:21:10.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:10 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:10.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:10 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:21:10.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:10 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:21:10.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:10 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:21:10.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:10 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:21:11.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101524 13.84 MB/sec execute 55 sec latency 46.518 ms 2024-04-25T04:21:11.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:11 smithi078 ceph-mon[21828]: Upgrade: Updating mgr.smithi078.eugwik 2024-04-25T04:21:11.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:11 smithi078 ceph-mon[21828]: Deploying daemon mgr.smithi078.eugwik on smithi078 2024-04-25T04:21:11.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:11 smithi112 ceph-mon[24160]: Upgrade: Updating mgr.smithi078.eugwik 2024-04-25T04:21:11.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:11 smithi112 ceph-mon[24160]: Deploying daemon mgr.smithi078.eugwik on smithi078 2024-04-25T04:21:12.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101976 13.84 MB/sec execute 56 sec latency 34.890 ms 2024-04-25T04:21:12.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:12 smithi078 ceph-mon[21828]: pgmap v32: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 772 op/s 2024-04-25T04:21:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:12 smithi112 ceph-mon[24160]: pgmap v32: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 772 op/s 2024-04-25T04:21:13.795 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102308 13.81 MB/sec execute 57 sec latency 51.133 ms 2024-04-25T04:21:14.415 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:14 smithi078 ceph-mon[21828]: pgmap v33: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 533 op/s 2024-04-25T04:21:14.415 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:14 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:14.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:14 smithi112 ceph-mon[24160]: pgmap v33: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 533 op/s 2024-04-25T04:21:14.877 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:14 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:14.878 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102692 13.77 MB/sec execute 58 sec latency 49.107 ms 2024-04-25T04:21:15.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102947 13.63 MB/sec execute 59 sec latency 44.059 ms 2024-04-25T04:21:16.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103263 13.55 MB/sec execute 60 sec latency 77.848 ms 2024-04-25T04:21:16.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:16 smithi078 ceph-mon[21828]: pgmap v34: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 758 op/s 2024-04-25T04:21:16.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:16 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:16.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:16 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:16.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:16 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:16.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:16 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:16.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:16 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:16 smithi112 ceph-mon[24160]: pgmap v34: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 758 op/s 2024-04-25T04:21:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:16 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:16 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:17.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:16 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:17.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:16 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:17.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:16 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:17.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103516 13.41 MB/sec execute 61 sec latency 52.333 ms 2024-04-25T04:21:18.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103804 13.35 MB/sec execute 62 sec latency 69.684 ms 2024-04-25T04:21:18.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:18 smithi112 ceph-mon[24160]: pgmap v35: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-04-25T04:21:18.978 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:18 smithi078 ceph-mon[21828]: pgmap v35: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-04-25T04:21:19.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104146 13.27 MB/sec execute 63 sec latency 54.051 ms 2024-04-25T04:21:20.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104481 13.11 MB/sec execute 64 sec latency 42.517 ms 2024-04-25T04:21:21.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:20 smithi078 ceph-mon[21828]: pgmap v36: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-04-25T04:21:21.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:20 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:21.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:20 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:21.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:20 smithi112 ceph-mon[24160]: pgmap v36: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-04-25T04:21:21.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:20 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:21.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:20 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:21.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104873 13.06 MB/sec execute 65 sec latency 33.104 ms 2024-04-25T04:21:22.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:22 smithi078 ceph-mon[21828]: pgmap v37: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 667 op/s 2024-04-25T04:21:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105280 13.07 MB/sec execute 66 sec latency 50.723 ms 2024-04-25T04:21:23.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:22 smithi112 ceph-mon[24160]: pgmap v37: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 667 op/s 2024-04-25T04:21:23.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105729 13.10 MB/sec execute 67 sec latency 48.329 ms 2024-04-25T04:21:24.333 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:24 smithi078 ceph-mon[21828]: pgmap v38: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-04-25T04:21:24.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:24 smithi112 ceph-mon[24160]: pgmap v38: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-04-25T04:21:24.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106076 13.07 MB/sec execute 68 sec latency 72.121 ms 2024-04-25T04:21:25.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106410 13.00 MB/sec execute 69 sec latency 41.941 ms 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: Standby manager daemon smithi078.eugwik restarted 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: Standby manager daemon smithi078.eugwik started 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/crt"}]: dispatch 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/key"}]: dispatch 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:21:26.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:26 smithi078 ceph-mon[21828]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.5 MiB/s wr, 641 op/s 2024-04-25T04:21:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: Standby manager daemon smithi078.eugwik restarted 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: Standby manager daemon smithi078.eugwik started 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/crt"}]: dispatch 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.eugwik/key"}]: dispatch 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:21:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:26 smithi112 ceph-mon[24160]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.5 MiB/s wr, 641 op/s 2024-04-25T04:21:26.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106767 12.96 MB/sec execute 70 sec latency 51.993 ms 2024-04-25T04:21:27.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:27 smithi078 ceph-mon[21828]: mgrmap e27: smithi112.lvtlnw(active, since 73s), standbys: smithi078.eugwik 2024-04-25T04:21:27.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:27 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:27.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:27 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:27 smithi112 ceph-mon[24160]: mgrmap e27: smithi112.lvtlnw(active, since 73s), standbys: smithi078.eugwik 2024-04-25T04:21:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:27 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:27 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:27.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107053 12.86 MB/sec execute 71 sec latency 42.627 ms 2024-04-25T04:21:28.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:28 smithi078 ceph-mon[21828]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-04-25T04:21:28.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:28 smithi112 ceph-mon[24160]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-04-25T04:21:28.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107394 12.83 MB/sec execute 72 sec latency 38.984 ms 2024-04-25T04:21:29.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:29 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:29.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:29 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:29.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107786 12.77 MB/sec execute 73 sec latency 57.372 ms 2024-04-25T04:21:30.040 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:21:30.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:30 smithi078 ceph-mon[21828]: pgmap v41: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-04-25T04:21:30.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:30 smithi112 ceph-mon[24160]: pgmap v41: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-04-25T04:21:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108202 12.68 MB/sec execute 74 sec latency 42.381 ms 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (46s) 5s ago 9m 17.6M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 5s ago 9m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (9m) 11s ago 9m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (38s) 5s ago 9m 45.4M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (3m) 5s ago 3m 39.5M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (3m) 11s ago 3m 13.3M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:21:31.152 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (16s) 5s ago 11m 110M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (93s) 11s ago 9m 505M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 676f050e5409 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 5s ago 11m 52.5M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (9m) 11s ago 9m 42.4M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (3m) 5s ago 3m 276M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (3m) 11s ago 3m 79.4M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (35s) 5s ago 9m 8208k - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (24s) 11s ago 8m 3493k - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 5s ago 8m 397M 1936M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 5s ago 7m 304M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 5s ago 7m 246M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 5s ago 6m 367M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (6m) 11s ago 6m 350M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (6m) 11s ago 6m 410M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (5m) 11s ago 5m 336M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:21:31.153 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (5m) 11s ago 5m 486M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:21:31.154 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (27s) 5s ago 8m 50.9M - 2.33.4 514e6a882f6e 31f3801c1259 2024-04-25T04:21:31.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:31 smithi078 ceph-mon[21828]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:31.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:31 smithi078 ceph-mon[21828]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:31.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:31 smithi112 ceph-mon[24160]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:31.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:31 smithi112 ceph-mon[24160]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:31.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108697 12.73 MB/sec execute 75 sec latency 33.554 ms 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:21:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:21:31.825 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:21:32.368 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons", 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:21:32.369 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:21:32.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:32 smithi078 ceph-mon[21828]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:32.453 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:32 smithi078 ceph-mon[21828]: pgmap v42: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-04-25T04:21:32.453 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:32 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/133208243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:21:32.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109255 12.81 MB/sec execute 76 sec latency 27.048 ms 2024-04-25T04:21:32.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:32 smithi112 ceph-mon[24160]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:32.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:32 smithi112 ceph-mon[24160]: pgmap v42: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-04-25T04:21:32.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:32 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/133208243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:21:32.976 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/505187524' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:33.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:33 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109665 12.79 MB/sec execute 77 sec latency 33.533 ms 2024-04-25T04:21:33.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:21:33.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/505187524' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:21:33.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:33.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:33.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:21:33.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:21:33.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:33.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:33 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' 2024-04-25T04:21:34.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:34.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:34.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:34.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: Failing over to other MGR 2024-04-25T04:21:34.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr fail", "who": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:21:34.471 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr fail", "who": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:21:34.471 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: osdmap e74: 8 total, 8 up, 8 in 2024-04-25T04:21:34.471 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 532 op/s 2024-04-25T04:21:34.471 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:34 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:21:34.686 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:34.686 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:34.686 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: Upgrade: Need to upgrade myself (mgr.smithi112.lvtlnw) 2024-04-25T04:21:34.686 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: Failing over to other MGR 2024-04-25T04:21:34.686 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: from='mgr.24421 172.21.15.112:0/412434492' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr fail", "who": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:21:34.686 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "mgr fail", "who": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:21:34.686 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: osdmap e74: 8 total, 8 up, 8 in 2024-04-25T04:21:34.687 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 532 op/s 2024-04-25T04:21:34.687 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:34 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:21:34.796 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110023 12.74 MB/sec execute 78 sec latency 38.660 ms 2024-04-25T04:21:35.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd='[{"prefix": "mgr fail", "who": "smithi112.lvtlnw"}]': finished 2024-04-25T04:21:35.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: mgrmap e28: smithi078.eugwik(active, starting, since 0.95345s), standbys: smithi112.lvtlnw 2024-04-25T04:21:35.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:21:35.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:21:35.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:21:35.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:21:35.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:35.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:35.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:21:35.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:35 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.24421 ' entity='mgr.smithi112.lvtlnw' cmd='[{"prefix": "mgr fail", "who": "smithi112.lvtlnw"}]': finished 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: mgrmap e28: smithi078.eugwik(active, starting, since 0.95345s), standbys: smithi112.lvtlnw 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:21:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:21:35.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:35 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:21:35.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110429 12.73 MB/sec execute 79 sec latency 34.795 ms 2024-04-25T04:21:36.498 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:36 smithi078 ceph-mon[21828]: mgrmap e29: smithi078.eugwik(active, since 2s), standbys: smithi112.lvtlnw 2024-04-25T04:21:36.498 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:36 smithi078 ceph-mon[21828]: pgmap v3: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:21:36.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:36 smithi112 ceph-mon[24160]: mgrmap e29: smithi078.eugwik(active, since 2s), standbys: smithi112.lvtlnw 2024-04-25T04:21:36.789 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:36 smithi112 ceph-mon[24160]: pgmap v3: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:21:36.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110803 12.68 MB/sec execute 80 sec latency 36.097 ms 2024-04-25T04:21:37.511 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:37 smithi078 ceph-mon[21828]: [25/Apr/2024:04:21:36] ENGINE Bus STARTING 2024-04-25T04:21:37.738 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:37 smithi112 ceph-mon[24160]: [25/Apr/2024:04:21:36] ENGINE Bus STARTING 2024-04-25T04:21:37.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111277 12.69 MB/sec execute 81 sec latency 55.733 ms 2024-04-25T04:21:38.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:38 smithi078 ceph-mon[21828]: [25/Apr/2024:04:21:36] ENGINE Serving on http://172.21.15.78:8765 2024-04-25T04:21:38.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:38 smithi078 ceph-mon[21828]: [25/Apr/2024:04:21:36] ENGINE Serving on https://172.21.15.78:7150 2024-04-25T04:21:38.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:38 smithi078 ceph-mon[21828]: [25/Apr/2024:04:21:36] ENGINE Bus STARTED 2024-04-25T04:21:38.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:38 smithi078 ceph-mon[21828]: mgrmap e30: smithi078.eugwik(active, since 4s), standbys: smithi112.lvtlnw 2024-04-25T04:21:38.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:38 smithi112 ceph-mon[24160]: [25/Apr/2024:04:21:36] ENGINE Serving on http://172.21.15.78:8765 2024-04-25T04:21:38.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:38 smithi112 ceph-mon[24160]: [25/Apr/2024:04:21:36] ENGINE Serving on https://172.21.15.78:7150 2024-04-25T04:21:38.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:38 smithi112 ceph-mon[24160]: [25/Apr/2024:04:21:36] ENGINE Bus STARTED 2024-04-25T04:21:38.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:38 smithi112 ceph-mon[24160]: mgrmap e30: smithi078.eugwik(active, since 4s), standbys: smithi112.lvtlnw 2024-04-25T04:21:38.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111579 12.57 MB/sec execute 82 sec latency 69.520 ms 2024-04-25T04:21:39.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:39 smithi078 ceph-mon[21828]: pgmap v5: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:21:39.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:39 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:39.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:39 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:39.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111893 12.52 MB/sec execute 83 sec latency 43.205 ms 2024-04-25T04:21:39.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:39 smithi112 ceph-mon[24160]: pgmap v5: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:21:39.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:39 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:39.856 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:39 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:40.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112158 12.48 MB/sec execute 84 sec latency 59.789 ms 2024-04-25T04:21:41.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:41 smithi078 ceph-mon[21828]: pgmap v6: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:21:41.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:41 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:41.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:41 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:41.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:41 smithi112 ceph-mon[24160]: pgmap v6: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-25T04:21:41.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:41 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:41.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:41 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:41.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112486 12.47 MB/sec execute 85 sec latency 85.984 ms 2024-04-25T04:21:42.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112785 12.43 MB/sec execute 86 sec latency 50.622 ms 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.6 MiB/s wr, 411 op/s 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:43.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:43 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:43.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113087 12.40 MB/sec execute 87 sec latency 48.726 ms 2024-04-25T04:21:43.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 3.6 MiB/s wr, 411 op/s 2024-04-25T04:21:43.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:21:43.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:21:43.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:21:43.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/2262392201' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:21:43.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:21:43.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:21:43.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:43.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:43 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:44.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:44 smithi078 ceph-mon[21828]: mgrmap e31: smithi078.eugwik(active, since 10s), standbys: smithi112.lvtlnw 2024-04-25T04:21:44.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113390 12.35 MB/sec execute 88 sec latency 60.563 ms 2024-04-25T04:21:44.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:44 smithi112 ceph-mon[24160]: mgrmap e31: smithi078.eugwik(active, since 10s), standbys: smithi112.lvtlnw 2024-04-25T04:21:45.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:45 smithi078 ceph-mon[21828]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.8 MiB/s wr, 318 op/s 2024-04-25T04:21:45.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:45 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:45.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:45 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:45.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113642 12.26 MB/sec execute 89 sec latency 58.023 ms 2024-04-25T04:21:45.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:45 smithi112 ceph-mon[24160]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.8 MiB/s wr, 318 op/s 2024-04-25T04:21:45.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:45 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:45.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:45 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:46.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114021 12.25 MB/sec execute 90 sec latency 46.634 ms 2024-04-25T04:21:47.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:47 smithi078 ceph-mon[21828]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.4 MiB/s wr, 452 op/s 2024-04-25T04:21:47.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114357 12.21 MB/sec execute 91 sec latency 69.362 ms 2024-04-25T04:21:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:47 smithi112 ceph-mon[24160]: pgmap v9: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.4 MiB/s wr, 452 op/s 2024-04-25T04:21:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114777 12.20 MB/sec execute 92 sec latency 64.190 ms 2024-04-25T04:21:49.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:49 smithi078 ceph-mon[21828]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-04-25T04:21:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:49 smithi112 ceph-mon[24160]: pgmap v10: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-04-25T04:21:49.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115185 12.13 MB/sec execute 93 sec latency 36.759 ms 2024-04-25T04:21:50.282 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:50 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:50.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:50 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:21:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115573 12.12 MB/sec execute 94 sec latency 39.532 ms 2024-04-25T04:21:51.299 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:51 smithi078 ceph-mon[21828]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-04-25T04:21:51.299 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:51 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:51.299 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:51 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:51.299 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:51 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:21:51.300 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:51 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:21:51.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:51 smithi112 ceph-mon[24160]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-04-25T04:21:51.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:51 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:51.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:51 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:51.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:51 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:21:51.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:51 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:21:51.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115931 12.12 MB/sec execute 95 sec latency 48.042 ms 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:52 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:52 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:21:52.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116351 12.13 MB/sec execute 96 sec latency 51.689 ms 2024-04-25T04:21:53.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:53 smithi078 ceph-mon[21828]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:21:53.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:53 smithi078 ceph-mon[21828]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:21:53.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:53 smithi078 ceph-mon[21828]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-04-25T04:21:53.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:53 smithi078 ceph-mon[21828]: pgmap v12: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.3 MiB/s wr, 648 op/s 2024-04-25T04:21:53.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:53 smithi078 ceph-mon[21828]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-04-25T04:21:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:53 smithi112 ceph-mon[24160]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:21:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:53 smithi112 ceph-mon[24160]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:21:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:53 smithi112 ceph-mon[24160]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-04-25T04:21:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:53 smithi112 ceph-mon[24160]: pgmap v12: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 6.3 MiB/s wr, 648 op/s 2024-04-25T04:21:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:53 smithi112 ceph-mon[24160]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-04-25T04:21:53.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116647 12.11 MB/sec execute 97 sec latency 52.729 ms 2024-04-25T04:21:54.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117006 12.07 MB/sec execute 98 sec latency 53.180 ms 2024-04-25T04:21:55.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:55 smithi078 ceph-mon[21828]: pgmap v13: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-04-25T04:21:55.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117470 12.07 MB/sec execute 99 sec latency 24.884 ms 2024-04-25T04:21:55.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:55 smithi112 ceph-mon[24160]: pgmap v13: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-04-25T04:21:56.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117883 12.06 MB/sec execute 100 sec latency 45.219 ms 2024-04-25T04:21:57.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:57 smithi078 ceph-mon[21828]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-04-25T04:21:57.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118437 12.07 MB/sec execute 101 sec latency 48.908 ms 2024-04-25T04:21:57.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:57 smithi112 ceph-mon[24160]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-04-25T04:21:58.797 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118877 12.02 MB/sec execute 102 sec latency 44.928 ms 2024-04-25T04:21:59.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:21:59 smithi078 ceph-mon[21828]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-04-25T04:21:59.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:21:59 smithi112 ceph-mon[24160]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-04-25T04:21:59.798 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119276 12.04 MB/sec execute 103 sec latency 47.378 ms 2024-04-25T04:22:00.798 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119643 12.03 MB/sec execute 104 sec latency 91.485 ms 2024-04-25T04:22:01.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:01 smithi078 ceph-mon[21828]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-04-25T04:22:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120028 12.05 MB/sec execute 105 sec latency 39.276 ms 2024-04-25T04:22:01.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:01 smithi112 ceph-mon[24160]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-04-25T04:22:02.798 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120326 12.02 MB/sec execute 106 sec latency 62.903 ms 2024-04-25T04:22:03.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:02 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:03.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:02 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:03.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:02 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:22:03.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:02 smithi078 ceph-mon[21828]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:22:03.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:02 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:03.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:02 smithi078 ceph-mon[21828]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-04-25T04:22:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:02 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:03.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:02 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:03.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:02 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:22:03.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:02 smithi112 ceph-mon[24160]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:22:03.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:02 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:03.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:02 smithi112 ceph-mon[24160]: pgmap v17: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-04-25T04:22:03.446 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:22:03.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120739 11.98 MB/sec execute 107 sec latency 53.605 ms 2024-04-25T04:22:04.300 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:04 smithi078 ceph-mon[21828]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:04.301 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:04 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:22:04.301 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:04 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:22:04.301 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:04 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:04.449 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:22:04.449 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (79s) 21s ago 10m 24.0M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:22:04.449 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 21s ago 10m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:22:04.449 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (9m) 25s ago 9m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:22:04.449 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (71s) 21s ago 10m 46.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (4m) 21s ago 4m 39.9M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (4m) 25s ago 4m 13.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (49s) 21s ago 11m 480M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283 running (2m) 25s ago 9m 81.2M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 676f050e5409 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 21s ago 11m 57.1M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (9m) 25s ago 9m 44.9M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (4m) 21s ago 4m 278M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (3m) 25s ago 3m 79.5M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (68s) 21s ago 10m 10.9M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (58s) 25s ago 9m 9.78M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 21s ago 8m 417M 1936M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 21s ago 8m 330M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 21s ago 7m 267M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:22:04.450 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 21s ago 7m 395M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:22:04.451 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (7m) 25s ago 7m 367M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:22:04.451 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (6m) 25s ago 6m 431M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:22:04.451 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (6m) 25s ago 6m 356M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:22:04.451 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (5m) 25s ago 5m 507M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:22:04.451 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 starting - - - - 2024-04-25T04:22:04.643 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:04 smithi112 ceph-mon[24160]: from='client.14816 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:04.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:04 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:22:04.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:04 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:22:04.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:04 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:04.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121343 12.04 MB/sec execute 108 sec latency 28.234 ms 2024-04-25T04:22:05.061 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:22:05.061 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:22:05.061 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:22:05.062 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:22:05.063 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:22:05.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:05 smithi078 ceph-mon[21828]: Upgrade: Updating mgr.smithi112.lvtlnw 2024-04-25T04:22:05.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:05 smithi078 ceph-mon[21828]: Deploying daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:22:05.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:05 smithi078 ceph-mon[21828]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:05.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:05 smithi078 ceph-mon[21828]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-04-25T04:22:05.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:05 smithi078 ceph-mon[21828]: from='client.24537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:05.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:05 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:05.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:05 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/67138741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons", 2024-04-25T04:22:05.604 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:22:05.605 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:22:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:05 smithi112 ceph-mon[24160]: Upgrade: Updating mgr.smithi112.lvtlnw 2024-04-25T04:22:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:05 smithi112 ceph-mon[24160]: Deploying daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:22:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:05 smithi112 ceph-mon[24160]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:05 smithi112 ceph-mon[24160]: pgmap v18: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-04-25T04:22:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:05 smithi112 ceph-mon[24160]: from='client.24537 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:05 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:05.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:05 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/67138741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:05.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121869 12.05 MB/sec execute 109 sec latency 32.013 ms 2024-04-25T04:22:06.231 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:22:06.331 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:06 smithi078 ceph-mon[21828]: from='client.? 172.21.15.78:0/3653023674' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:22:06.517 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:06 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/3653023674' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:22:06.805 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122379 12.01 MB/sec execute 110 sec latency 36.500 ms 2024-04-25T04:22:07.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:07 smithi078 ceph-mon[21828]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:07.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:07 smithi078 ceph-mon[21828]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 748 op/s 2024-04-25T04:22:07.571 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:07 smithi112 ceph-mon[24160]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:07.571 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:07 smithi112 ceph-mon[24160]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 748 op/s 2024-04-25T04:22:07.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122717 12.00 MB/sec execute 111 sec latency 49.839 ms 2024-04-25T04:22:08.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123179 12.04 MB/sec execute 112 sec latency 54.665 ms 2024-04-25T04:22:09.436 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:09 smithi078 ceph-mon[21828]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-04-25T04:22:09.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:09 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:09 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:09 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:09.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:09 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:09 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:09 smithi112 ceph-mon[24160]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-04-25T04:22:09.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:09 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:09 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:09 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:09.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:09 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:09 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:09.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123521 12.04 MB/sec execute 113 sec latency 47.304 ms 2024-04-25T04:22:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123715 11.98 MB/sec execute 114 sec latency 94.314 ms 2024-04-25T04:22:11.483 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:11 smithi078 ceph-mon[21828]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-04-25T04:22:11.488 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:11 smithi112 ceph-mon[24160]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-04-25T04:22:11.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124054 11.95 MB/sec execute 115 sec latency 79.211 ms 2024-04-25T04:22:12.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124461 11.91 MB/sec execute 116 sec latency 39.772 ms 2024-04-25T04:22:13.500 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:13 smithi112 ceph-mon[24160]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 774 op/s 2024-04-25T04:22:13.536 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:13 smithi078 ceph-mon[21828]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 774 op/s 2024-04-25T04:22:13.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124755 11.88 MB/sec execute 117 sec latency 39.521 ms 2024-04-25T04:22:14.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125069 11.87 MB/sec execute 118 sec latency 59.887 ms 2024-04-25T04:22:14.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:14 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:14.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:14 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:14.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:14 smithi078 ceph-mon[21828]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-04-25T04:22:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:14 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:14 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:15.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:14 smithi112 ceph-mon[24160]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-04-25T04:22:15.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125407 11.84 MB/sec execute 119 sec latency 60.603 ms 2024-04-25T04:22:16.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:16 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:16.489 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:16 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:16.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:16 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:16.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:16 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:16.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125762 11.77 MB/sec execute 120 sec latency 57.492 ms 2024-04-25T04:22:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:17 smithi078 ceph-mon[21828]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 702 op/s 2024-04-25T04:22:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:17 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:17 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:17.454 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:17 smithi112 ceph-mon[24160]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 702 op/s 2024-04-25T04:22:17.454 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:17 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:17.454 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:17 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:17.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126356 11.82 MB/sec execute 121 sec latency 67.103 ms 2024-04-25T04:22:18.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:18 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:22:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:18 smithi078 ceph-mon[21828]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:22:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:18 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:22:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:18 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:22:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:18 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:22:18.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:18 smithi078 ceph-mon[21828]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:22:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:18 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:22:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:18 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:22:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:18 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:22:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:18 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:22:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:18 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:22:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:18 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/103375169' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:22:18.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127191 11.96 MB/sec execute 122 sec latency 30.500 ms 2024-04-25T04:22:19.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:19 smithi078 ceph-mon[21828]: mgrmap e32: smithi078.eugwik(active, since 44s), standbys: smithi112.lvtlnw 2024-04-25T04:22:19.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:19 smithi078 ceph-mon[21828]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-04-25T04:22:19.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:19 smithi112 ceph-mon[24160]: mgrmap e32: smithi078.eugwik(active, since 44s), standbys: smithi112.lvtlnw 2024-04-25T04:22:19.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:19 smithi112 ceph-mon[24160]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-04-25T04:22:19.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127464 11.90 MB/sec execute 123 sec latency 56.351 ms 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.eugwik"}]: dispatch 2024-04-25T04:22:20.423 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.eugwik"}]': finished 2024-04-25T04:22:20.424 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.lvtlnw"}]: dispatch 2024-04-25T04:22:20.424 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.lvtlnw"}]': finished 2024-04-25T04:22:20.424 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:20 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-25T04:22:20.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.eugwik"}]: dispatch 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.eugwik"}]': finished 2024-04-25T04:22:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.lvtlnw"}]: dispatch 2024-04-25T04:22:20.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi112.lvtlnw"}]': finished 2024-04-25T04:22:20.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:20 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-25T04:22:20.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127796 11.87 MB/sec execute 124 sec latency 66.401 ms 2024-04-25T04:22:21.447 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:21 smithi078 ceph-mon[21828]: Upgrade: Setting container_image for all mgr 2024-04-25T04:22:21.447 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:21 smithi078 ceph-mon[21828]: pgmap v26: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-04-25T04:22:21.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:21 smithi112 ceph-mon[24160]: Upgrade: Setting container_image for all mgr 2024-04-25T04:22:21.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:21 smithi112 ceph-mon[24160]: pgmap v26: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.4 MiB/s wr, 451 op/s 2024-04-25T04:22:21.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128077 11.83 MB/sec execute 125 sec latency 41.323 ms 2024-04-25T04:22:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128373 11.83 MB/sec execute 126 sec latency 35.244 ms 2024-04-25T04:22:23.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:23 smithi078 ceph-mon[21828]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 736 op/s 2024-04-25T04:22:23.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:23 smithi078 ceph-mon[21828]: Upgrade: Updating mon.smithi078 2024-04-25T04:22:23.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:23 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:23.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:23 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:22:23.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:23 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:22:23.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:23 smithi078 ceph-mon[21828]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:23.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:23 smithi078 ceph-mon[21828]: Deploying daemon mon.smithi078 on smithi078 2024-04-25T04:22:23.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128685 11.79 MB/sec execute 127 sec latency 51.703 ms 2024-04-25T04:22:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:23 smithi112 ceph-mon[24160]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 736 op/s 2024-04-25T04:22:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:23 smithi112 ceph-mon[24160]: Upgrade: Updating mon.smithi078 2024-04-25T04:22:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:23 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:23 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:22:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:23 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:22:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:23 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:23.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:23 smithi112 ceph-mon[24160]: Deploying daemon mon.smithi078 on smithi078 2024-04-25T04:22:24.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129053 11.76 MB/sec execute 128 sec latency 69.227 ms 2024-04-25T04:22:25.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:25 smithi078 ceph-mon[21828]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 489 op/s 2024-04-25T04:22:25.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:25 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:22:25.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129399 11.71 MB/sec execute 129 sec latency 49.528 ms 2024-04-25T04:22:25.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:25 smithi112 ceph-mon[24160]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 489 op/s 2024-04-25T04:22:26.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[21824]: 2024-04-25T04:22:25.888+0000 7f876f5c3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:22:26.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[21824]: 2024-04-25T04:22:25.888+0000 7f876f5c3700 -1 mon.smithi078@0(leader) e2 *** Got Signal Terminated *** 2024-04-25T04:22:26.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:25 smithi078 podman[62774]: 2024-04-25 04:22:25.910250562 +0000 UTC m=+0.189807799 container died 74117070b5c0cc237da0cfed9ae6c06df00bc650df578d92aa6f0d3d1028ff92 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, vendor=Red Hat, Inc., name=centos-stream, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, version=8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public) 2024-04-25T04:22:26.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:26 smithi078 podman[62774]: 2024-04-25 04:22:26.73223639 +0000 UTC m=+1.011793614 container cleanup 74117070b5c0cc237da0cfed9ae6c06df00bc650df578d92aa6f0d3d1028ff92 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, name=centos-stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-04-25T04:22:26.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:26 smithi078 bash[62774]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078 2024-04-25T04:22:26.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129692 11.68 MB/sec execute 130 sec latency 40.511 ms 2024-04-25T04:22:27.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:27 smithi078 podman[62788]: 2024-04-25 04:22:27.098626007 +0000 UTC m=+1.182540650 container remove 74117070b5c0cc237da0cfed9ae6c06df00bc650df578d92aa6f0d3d1028ff92 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, distribution-scope=public, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-type=git, name=centos-stream, ceph=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-25T04:22:27.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:27 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-04-25T04:22:27.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:27 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:22:27.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:27 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service: Consumed 15.438s CPU time. 2024-04-25T04:22:27.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130041 11.71 MB/sec execute 131 sec latency 77.983 ms 2024-04-25T04:22:27.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:27 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:22:28.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:27 smithi078 podman[62924]: 2024-04-25 04:22:27.796433837 +0000 UTC m=+0.020584414 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:22:28.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 podman[62924]: 2024-04-25 04:22:28.012894661 +0000 UTC m=+0.237045225 container create 261e9ec88b00572e9f96cbd75ea322a28d6e8af4d669a88db2190ccf6e427276 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:22:28.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 podman[62924]: 2024-04-25 04:22:28.350273089 +0000 UTC m=+0.574423654 container init 261e9ec88b00572e9f96cbd75ea322a28d6e8af4d669a88db2190ccf6e427276 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-25T04:22:28.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 podman[62924]: 2024-04-25 04:22:28.355402964 +0000 UTC m=+0.579553539 container start 261e9ec88b00572e9f96cbd75ea322a28d6e8af4d669a88db2190ccf6e427276 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-25T04:22:28.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: set uid:gid to 167:167 (ceph:ceph) 2024-04-25T04:22:28.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev), process ceph-mon, pid 2 2024-04-25T04:22:28.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: pidfile_write: ignore empty --pid-file 2024-04-25T04:22:28.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: load: jerasure load: lrc 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: RocksDB version: 7.9.2 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Git sha 0 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Compile date 2024-04-25 01:05:22 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: DB SUMMARY 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: DB Session ID: TTFCCZ8RVNG67ITC7DBW 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: CURRENT file: CURRENT 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: IDENTITY file: IDENTITY 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000030.sst 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000028.log size: 1566435 ; 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.error_if_exists: 0 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.create_if_missing: 0 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.paranoid_checks: 1 2024-04-25T04:22:28.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-25T04:22:28.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-25T04:22:28.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-25T04:22:28.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.env: 0x557ca4a66cc0 2024-04-25T04:22:28.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.fs: PosixFileSystem 2024-04-25T04:22:28.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.info_log: 0x557ca632bc60 2024-04-25T04:22:28.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_file_opening_threads: 16 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.statistics: (nil) 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.use_fsync: 0 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_log_file_size: 0 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.keep_log_file_num: 1000 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.recycle_log_file_num: 0 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.allow_fallocate: 1 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.allow_mmap_reads: 0 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.allow_mmap_writes: 0 2024-04-25T04:22:28.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.use_direct_reads: 0 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.create_missing_column_families: 0 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.db_log_dir: 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.wal_dir: 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.advise_random_on_open: 1 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.db_write_buffer_size: 0 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.write_buffer_manager: 0x557ca62fd0e0 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-25T04:22:28.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.rate_limiter: (nil) 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.wal_recovery_mode: 2 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.enable_thread_tracking: 0 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.enable_pipelined_write: 0 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.unordered_write: 0 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.row_cache: None 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.wal_filter: None 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-25T04:22:28.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.allow_ingest_behind: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.two_write_queues: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.manual_wal_flush: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.wal_compression: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.atomic_flush: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.log_readahead_size: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.best_efforts_recovery: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.allow_data_in_errors: 0 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.db_host_id: __hostname__ 2024-04-25T04:22:28.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_background_jobs: 2 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_background_compactions: -1 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_subcompactions: 1 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_total_wal_size: 0 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_open_files: -1 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bytes_per_sync: 0 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-25T04:22:28.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_readahead_size: 0 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_background_flushes: -1 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Compression algorithms supported: 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kZSTD supported: 0 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kXpressCompression supported: 0 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kLZ4HCCompression supported: 1 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kZlibCompression supported: 1 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kSnappyCompression supported: 1 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kLZ4Compression supported: 1 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: kBZip2Compression supported: 0 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-25T04:22:28.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.merge_operator: 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_filter: None 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_filter_factory: None 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.sst_partitioner_factory: None 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557ca632be20) 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-04-25T04:22:28.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 4 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x557ca632f1f0 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-04-25T04:22:28.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: bloomfilter 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 5 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_auto_readahead_size: 262144 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: prepopulate_block_cache: 0 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: initial_auto_readahead_size: 8192 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_file_reads_for_auto_readahead: 2 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.write_buffer_size: 33554432 2024-04-25T04:22:28.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_write_buffer_number: 2 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression: NoCompression 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression: Disabled 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.prefix_extractor: nullptr 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.num_levels: 7 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-25T04:22:28.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.level: 32767 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.strategy: 0 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.enabled: false 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-25T04:22:28.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.target_file_size_base: 67108864 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-25T04:22:28.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.arena_block_size: 1048576 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.disable_auto_compactions: 0 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-25T04:22:28.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.inplace_update_support: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.bloom_locality: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.max_successive_merges: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.paranoid_file_checks: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.force_consistency_checks: 1 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.report_bg_io_stats: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.ttl: 2592000 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-25T04:22:28.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-25T04:22:28.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-25T04:22:28.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.enable_blob_files: false 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.min_blob_size: 0 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.blob_file_size: 268435456 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.blob_file_starting_level: 0 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11887, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-25T04:22:28.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6fcc9ba9-da85-472e-baba-c561b807e90e 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018948388656, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018948394960, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1389562, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11888, "largest_seqno": 12710, "table_properties": {"data_size": 1385198, "index_size": 2428, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 7964, "raw_average_key_size": 23, "raw_value_size": 1377534, "raw_average_value_size": 4004, "num_data_blocks": 111, "num_entries": 344, "num_filter_entries": 344, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714018948, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6fcc9ba9-da85-472e-baba-c561b807e90e", "db_session_id": "TTFCCZ8RVNG67ITC7DBW", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018948395080, "job": 1, "event": "recovery_finished"} 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557ca644c000 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: DB pointer 0x557ca6436000 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-25T04:22:28.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 1/0 1.33 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 398.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L6 1/0 7.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 8.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 398.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 398.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-04-25T04:22:28.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 398.6 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 122.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 122.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:22:28.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache BinnedLRUCache@0x557ca632f1f0#2 capacity: 512.00 MB usage: 3.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.58 KB,0.000491738%) Misc(1,0.00 KB,0%) 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???) e2 preinit fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???).mds e7 new map 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???).mds e7 print_map 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e7 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: 1 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Filesystem 'foofs' (1) 2024-04-25T04:22:28.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: fs_name foofs 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: epoch 5 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: created 2024-04-25T04:17:37.837625+0000 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: modified 2024-04-25T04:17:43.857514+0000 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: tableserver 0 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: root 0 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_timeout 60 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_autoclose 300 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_file_size 1099511627776 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_xattr_size 65536 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: required_client_features {} 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure 0 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure_osd_epoch 0 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-25T04:22:28.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_mds 1 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: in 0 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: up {0=14498} 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: failed 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: damaged 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: stopped 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_pools [3] 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_pool 2 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: inline_data disabled 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: balancer 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: bal_rank_mask -1 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: standby_count_wanted 1 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: qdb_cluster leader: 0 members: 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi078.nlrtle{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/131251842,v1:172.21.15.78:6835/131251842] compat {c=[1],r=[1],i=[7ff]}] 2024-04-25T04:22:28.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Standby daemons: 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi112.xminls{-1:24271} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] compat {c=[1],r=[1],i=[7ff]}] 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???).osd e74 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???).osd e74 crush map has features 432629239337189376, adjusting msgr requires 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 ceph-mon[62942]: mon.smithi078@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 bash[62924]: 261e9ec88b00572e9f96cbd75ea322a28d6e8af4d669a88db2190ccf6e427276 2024-04-25T04:22:28.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:28 smithi078 systemd[1]: Started Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:22:28.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130445 11.69 MB/sec execute 132 sec latency 42.223 ms 2024-04-25T04:22:29.119 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:22:29.119 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:22:29.119 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-25T04:22:29.119 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: pgmap v30: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-04-25T04:22:29.119 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: mon.smithi078 calling monitor election 2024-04-25T04:22:29.119 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: mon.smithi078 is new leader, mons smithi078,smithi112 in quorum (ranks 0,1) 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: monmap epoch 2 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: last_changed 2024-04-25T04:12:27.641249+0000 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: created 2024-04-25T04:10:09.587319+0000 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: min_mon_release 17 (quincy) 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: election_strategy: 1 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: 1: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: osdmap e74: 8 total, 8 up, 8 in 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: mgrmap e32: smithi078.eugwik(active, since 55s), standbys: smithi112.lvtlnw 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: overall HEALTH_OK 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: mgrmap e33: smithi078.eugwik(active, since 55s), standbys: smithi112.lvtlnw 2024-04-25T04:22:29.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:29 smithi078 ceph-mon[62942]: from='mgr.14764 ' entity='' 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: from='mgr.14764 172.21.15.78:0/4021432424' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: pgmap v30: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: mon.smithi078 calling monitor election 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: mon.smithi078 is new leader, mons smithi078,smithi112 in quorum (ranks 0,1) 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: monmap epoch 2 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: last_changed 2024-04-25T04:12:27.641249+0000 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: created 2024-04-25T04:10:09.587319+0000 2024-04-25T04:22:29.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: min_mon_release 17 (quincy) 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: election_strategy: 1 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: 1: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: osdmap e74: 8 total, 8 up, 8 in 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: mgrmap e32: smithi078.eugwik(active, since 55s), standbys: smithi112.lvtlnw 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: overall HEALTH_OK 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: mgrmap e33: smithi078.eugwik(active, since 55s), standbys: smithi112.lvtlnw 2024-04-25T04:22:29.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:29 smithi112 ceph-mon[24160]: from='mgr.14764 ' entity='' 2024-04-25T04:22:29.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130853 11.72 MB/sec execute 133 sec latency 36.623 ms 2024-04-25T04:22:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131316 11.69 MB/sec execute 134 sec latency 25.366 ms 2024-04-25T04:22:31.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:31 smithi078 ceph-mon[62942]: mgrmap e34: smithi078.eugwik(active, since 56s), standbys: smithi112.lvtlnw 2024-04-25T04:22:31.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:31 smithi112 ceph-mon[24160]: mgrmap e34: smithi078.eugwik(active, since 56s), standbys: smithi112.lvtlnw 2024-04-25T04:22:31.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131612 11.66 MB/sec execute 135 sec latency 48.458 ms 2024-04-25T04:22:32.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131845 11.63 MB/sec execute 136 sec latency 77.306 ms 2024-04-25T04:22:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132104 11.60 MB/sec execute 137 sec latency 69.808 ms 2024-04-25T04:22:34.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132414 11.59 MB/sec execute 138 sec latency 87.873 ms 2024-04-25T04:22:35.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132743 11.53 MB/sec execute 139 sec latency 72.352 ms 2024-04-25T04:22:36.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133036 11.49 MB/sec execute 140 sec latency 29.623 ms 2024-04-25T04:22:37.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:37 smithi078 ceph-mon[62942]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:22:37.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:37 smithi078 ceph-mon[62942]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:22:37.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:37 smithi078 ceph-mon[62942]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:22:37.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:37 smithi078 ceph-mon[62942]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:22:37.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:37 smithi078 ceph-mon[62942]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:22:37.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:37 smithi078 ceph-mon[62942]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:22:37.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133315 11.48 MB/sec execute 141 sec latency 90.637 ms 2024-04-25T04:22:37.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:37 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw restarted 2024-04-25T04:22:37.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:37 smithi112 ceph-mon[24160]: Standby manager daemon smithi112.lvtlnw started 2024-04-25T04:22:37.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:37 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/crt"}]: dispatch 2024-04-25T04:22:37.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:37 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-25T04:22:37.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:37 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi112.lvtlnw/key"}]: dispatch 2024-04-25T04:22:37.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:37 smithi112 ceph-mon[24160]: from='mgr.? 172.21.15.112:0/3179789143' entity='mgr.smithi112.lvtlnw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-25T04:22:38.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: mgrmap e35: smithi078.eugwik(active, since 64s), standbys: smithi112.lvtlnw 2024-04-25T04:22:38.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: Active manager daemon smithi078.eugwik restarted 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: Activating manager daemon smithi078.eugwik 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: osdmap e75: 8 total, 8 up, 8 in 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: mgrmap e36: smithi078.eugwik(active, starting, since 0.00728095s), standbys: smithi112.lvtlnw 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:22:38.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:22:38.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:22:38.814 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133686 11.48 MB/sec execute 142 sec latency 50.682 ms 2024-04-25T04:22:39.105 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:22:39.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: mgrmap e35: smithi078.eugwik(active, since 64s), standbys: smithi112.lvtlnw 2024-04-25T04:22:39.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: Active manager daemon smithi078.eugwik restarted 2024-04-25T04:22:39.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: Activating manager daemon smithi078.eugwik 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: osdmap e75: 8 total, 8 up, 8 in 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: mgrmap e36: smithi078.eugwik(active, starting, since 0.00728095s), standbys: smithi112.lvtlnw 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi078.eugwik", "id": "smithi078.eugwik"}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr metadata", "who": "smithi112.lvtlnw", "id": "smithi112.lvtlnw"}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:22:39.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: Manager daemon smithi078.eugwik is now available 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/mirror_snapshot_schedule"}]: dispatch 2024-04-25T04:22:39.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:38 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.eugwik/trash_purge_schedule"}]: dispatch 2024-04-25T04:22:39.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134130 11.50 MB/sec execute 143 sec latency 27.185 ms 2024-04-25T04:22:40.108 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:22:40.108 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (115s) 23s ago 11m 26.1M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:22:40.108 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (11m) 23s ago 11m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:22:40.108 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (10m) 26s ago 10m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (107s) 23s ago 10m 46.9M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (4m) 23s ago 4m 40.1M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (4m) 26s ago 4m 13.8M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (85s) 23s ago 12m 496M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (32s) 26s ago 10m 111M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (12m) 23s ago 12m 57.3M 2048M 17.2.0 e1d6a67b021e 74117070b5c0 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (10m) 26s ago 10m 44.3M 2048M 17.2.0 e1d6a67b021e 179da40ce14c 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (4m) 23s ago 4m 273M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (4m) 26s ago 4m 81.2M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (104s) 23s ago 10m 10.1M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (93s) 26s ago 10m 10.6M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 23s ago 9m 443M 1936M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 23s ago 8m 394M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 23s ago 8m 277M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 23s ago 7m 415M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:22:40.109 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (7m) 26s ago 7m 412M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:22:40.110 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (7m) 26s ago 7m 473M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:22:40.110 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (6m) 26s ago 6m 410M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:22:40.110 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (6m) 26s ago 6m 551M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:22:40.110 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (39s) 23s ago 9m 49.8M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:22:40.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:40 smithi112 ceph-mon[24160]: [25/Apr/2024:04:22:38] ENGINE Bus STARTING 2024-04-25T04:22:40.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:40 smithi112 ceph-mon[24160]: [25/Apr/2024:04:22:38] ENGINE Serving on https://172.21.15.78:7150 2024-04-25T04:22:40.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:40 smithi112 ceph-mon[24160]: [25/Apr/2024:04:22:38] ENGINE Serving on http://172.21.15.78:8765 2024-04-25T04:22:40.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:40 smithi112 ceph-mon[24160]: [25/Apr/2024:04:22:38] ENGINE Bus STARTED 2024-04-25T04:22:40.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:40 smithi112 ceph-mon[24160]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:40.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:40 smithi112 ceph-mon[24160]: mgrmap e37: smithi078.eugwik(active, since 1.01876s), standbys: smithi112.lvtlnw 2024-04-25T04:22:40.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:40 smithi112 ceph-mon[24160]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:40.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:40 smithi078 ceph-mon[62942]: [25/Apr/2024:04:22:38] ENGINE Bus STARTING 2024-04-25T04:22:40.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:40 smithi078 ceph-mon[62942]: [25/Apr/2024:04:22:38] ENGINE Serving on https://172.21.15.78:7150 2024-04-25T04:22:40.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:40 smithi078 ceph-mon[62942]: [25/Apr/2024:04:22:38] ENGINE Serving on http://172.21.15.78:8765 2024-04-25T04:22:40.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:40 smithi078 ceph-mon[62942]: [25/Apr/2024:04:22:38] ENGINE Bus STARTED 2024-04-25T04:22:40.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:40 smithi078 ceph-mon[62942]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:40.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:40 smithi078 ceph-mon[62942]: mgrmap e37: smithi078.eugwik(active, since 1.01876s), standbys: smithi112.lvtlnw 2024-04-25T04:22:40.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:40 smithi078 ceph-mon[62942]: pgmap v3: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:22:40.686 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 3 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:22:40.687 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:22:40.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134495 11.49 MB/sec execute 144 sec latency 31.863 ms 2024-04-25T04:22:41.227 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:22:41.227 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:22:41.227 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:22:41.227 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:22:41.228 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:22:41.228 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-04-25T04:22:41.228 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:22:41.228 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-04-25T04:22:41.228 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-04-25T04:22:41.228 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:22:41.228 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:22:41.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:41 smithi078 ceph-mon[62942]: from='client.24581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:41.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:41 smithi078 ceph-mon[62942]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:41.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:41 smithi078 ceph-mon[62942]: mgrmap e38: smithi078.eugwik(active, since 2s), standbys: smithi112.lvtlnw 2024-04-25T04:22:41.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:41 smithi078 ceph-mon[62942]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:41.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:41 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/226178418' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:41.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:41 smithi112 ceph-mon[24160]: from='client.24581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:41 smithi112 ceph-mon[24160]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:41 smithi112 ceph-mon[24160]: mgrmap e38: smithi078.eugwik(active, since 2s), standbys: smithi112.lvtlnw 2024-04-25T04:22:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:41 smithi112 ceph-mon[24160]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:41 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/226178418' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:41.755 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:22:41.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134879 11.47 MB/sec execute 145 sec latency 41.414 ms 2024-04-25T04:22:42.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:42 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/346057469' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:22:42.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:42 smithi112 ceph-mon[24160]: from='client.? 172.21.15.78:0/346057469' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:22:42.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135111 11.43 MB/sec execute 146 sec latency 72.355 ms 2024-04-25T04:22:43.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:43 smithi078 ceph-mon[62942]: from='client.24589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:43.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:43 smithi078 ceph-mon[62942]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:43.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:43 smithi078 ceph-mon[62942]: mgrmap e39: smithi078.eugwik(active, since 4s), standbys: smithi112.lvtlnw 2024-04-25T04:22:43.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:43 smithi112 ceph-mon[24160]: from='client.24589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:22:43.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:43 smithi112 ceph-mon[24160]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:43.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:43 smithi112 ceph-mon[24160]: mgrmap e39: smithi078.eugwik(active, since 4s), standbys: smithi112.lvtlnw 2024-04-25T04:22:43.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135345 11.40 MB/sec execute 147 sec latency 68.917 ms 2024-04-25T04:22:44.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135568 11.37 MB/sec execute 148 sec latency 63.885 ms 2024-04-25T04:22:44.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:44.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:44.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:44 smithi078 ceph-mon[62942]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:45.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:44 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:44 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:45.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:44 smithi112 ceph-mon[24160]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-25T04:22:45.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135813 11.35 MB/sec execute 149 sec latency 82.101 ms 2024-04-25T04:22:46.339 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:46 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:46.339 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:46 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:46.339 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:46 smithi112 ceph-mon[24160]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 3.7 MiB/s wr, 361 op/s 2024-04-25T04:22:46.475 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:46.475 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:46.475 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:46 smithi078 ceph-mon[62942]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 3.7 MiB/s wr, 361 op/s 2024-04-25T04:22:46.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136264 11.32 MB/sec execute 150 sec latency 40.264 ms 2024-04-25T04:22:47.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136603 11.29 MB/sec execute 151 sec latency 53.916 ms 2024-04-25T04:22:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136960 11.30 MB/sec execute 152 sec latency 51.459 ms 2024-04-25T04:22:48.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:48 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:48.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:48 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:48.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:48 smithi112 ceph-mon[24160]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 280 op/s 2024-04-25T04:22:49.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:48 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:49.054 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:48 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:49.054 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:48 smithi078 ceph-mon[62942]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 280 op/s 2024-04-25T04:22:49.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137348 11.31 MB/sec execute 153 sec latency 43.803 ms 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:22:50.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:22:50.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:22:50.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:22:50.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:22:50.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:50 smithi078 ceph-mon[62942]: pgmap v9: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.4 MiB/s wr, 449 op/s 2024-04-25T04:22:50.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: Updating smithi078:/etc/ceph/ceph.conf 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.conf 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: Updating smithi112:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-04-25T04:22:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:50 smithi112 ceph-mon[24160]: pgmap v9: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.4 MiB/s wr, 449 op/s 2024-04-25T04:22:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137716 11.32 MB/sec execute 154 sec latency 51.151 ms 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:51.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-25T04:22:51.730 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: Updating smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:22:51.730 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: Updating smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-25T04:22:51.730 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:22:51.731 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:51 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-25T04:22:51.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137977 11.29 MB/sec execute 155 sec latency 46.279 ms 2024-04-25T04:22:52.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:52 smithi078 ceph-mon[62942]: pgmap v10: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-04-25T04:22:52.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138387 11.27 MB/sec execute 156 sec latency 35.876 ms 2024-04-25T04:22:52.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:52 smithi112 ceph-mon[24160]: pgmap v10: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-04-25T04:22:53.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:53 smithi078 ceph-mon[62942]: Upgrade: Updating mon.smithi112 2024-04-25T04:22:53.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:53.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:22:53.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:22:53.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:53.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:53 smithi078 ceph-mon[62942]: Deploying daemon mon.smithi112 on smithi112 2024-04-25T04:22:53.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:53.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138716 11.25 MB/sec execute 157 sec latency 47.740 ms 2024-04-25T04:22:53.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:53 smithi112 ceph-mon[24160]: Upgrade: Updating mon.smithi112 2024-04-25T04:22:53.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:53 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:53.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:53 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:22:53.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:53 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:22:53.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:53 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:22:53.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:53 smithi112 ceph-mon[24160]: Deploying daemon mon.smithi112 on smithi112 2024-04-25T04:22:53.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:53 smithi112 ceph-mon[24160]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:22:54.706 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:54 smithi078 ceph-mon[62942]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-04-25T04:22:54.809 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:54 smithi112 systemd[1]: Stopping Ceph mon.smithi112 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:22:54.810 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:54 smithi112 ceph-mon[24160]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-04-25T04:22:54.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139030 11.25 MB/sec execute 158 sec latency 45.818 ms 2024-04-25T04:22:55.127 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:54 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112[24150]: 2024-04-25T04:22:54.802+0000 7fdb3d1b7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi112 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:22:55.127 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:54 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112[24150]: 2024-04-25T04:22:54.802+0000 7fdb3d1b7700 -1 mon.smithi112@1(peon) e2 *** Got Signal Terminated *** 2024-04-25T04:22:55.127 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:54 smithi112 podman[49847]: 2024-04-25 04:22:54.823541315 +0000 UTC m=+0.177604521 container died 179da40ce14ca95e5fa5450bf64a21a5e54158a3957e43b7450c0e45aa9eb5e1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, release=754, CEPH_POINT_RELEASE=-17.2.0, version=8, vcs-type=git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, distribution-scope=public, maintainer=Guillaume Abrioux ) 2024-04-25T04:22:55.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:55 smithi112 podman[49847]: 2024-04-25 04:22:55.2124091 +0000 UTC m=+0.566472313 container cleanup 179da40ce14ca95e5fa5450bf64a21a5e54158a3957e43b7450c0e45aa9eb5e1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870) 2024-04-25T04:22:55.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:55 smithi112 bash[49847]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112 2024-04-25T04:22:55.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139570 11.26 MB/sec execute 159 sec latency 44.583 ms 2024-04-25T04:22:55.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:55 smithi112 podman[49861]: 2024-04-25 04:22:55.464161985 +0000 UTC m=+0.639079569 container remove 179da40ce14ca95e5fa5450bf64a21a5e54158a3957e43b7450c0e45aa9eb5e1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, name=centos-stream, version=8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., architecture=x86_64, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-25T04:22:55.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:55 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi112.service: Deactivated successfully. 2024-04-25T04:22:55.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:55 smithi112 systemd[1]: Stopped Ceph mon.smithi112 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:22:55.832 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:55 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi112.service: Consumed 6.744s CPU time. 2024-04-25T04:22:56.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:55 smithi112 systemd[1]: Starting Ceph mon.smithi112 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:22:56.103 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 podman[49989]: 2024-04-25 04:22:55.956317819 +0000 UTC m=+0.021605051 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:22:56.376 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 podman[49989]: 2024-04-25 04:22:56.098562601 +0000 UTC m=+0.163849832 container create cf109ce58bbce47b69b86cf9f1123581e047845b354ccae8611cd9f3e9baa214 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:22:56.376 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 podman[49989]: 2024-04-25 04:22:56.34362163 +0000 UTC m=+0.408908860 container init cf109ce58bbce47b69b86cf9f1123581e047845b354ccae8611cd9f3e9baa214 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:22:56.376 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 podman[49989]: 2024-04-25 04:22:56.346874662 +0000 UTC m=+0.412161894 container start cf109ce58bbce47b69b86cf9f1123581e047845b354ccae8611cd9f3e9baa214 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:22:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: set uid:gid to 167:167 (ceph:ceph) 2024-04-25T04:22:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev), process ceph-mon, pid 2 2024-04-25T04:22:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: pidfile_write: ignore empty --pid-file 2024-04-25T04:22:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: load: jerasure load: lrc 2024-04-25T04:22:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: RocksDB version: 7.9.2 2024-04-25T04:22:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Git sha 0 2024-04-25T04:22:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Compile date 2024-04-25 01:05:22 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: DB SUMMARY 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: DB Session ID: 718BOIJCDRJG2KDGNXVX 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: CURRENT file: CURRENT 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: IDENTITY file: IDENTITY 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi112/store.db dir, Total Num: 1, files: 000024.sst 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi112/store.db: 000022.log size: 5642165 ; 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.error_if_exists: 0 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.create_if_missing: 0 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.paranoid_checks: 1 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.env: 0x5571d4bf7cc0 2024-04-25T04:22:56.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.fs: PosixFileSystem 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.info_log: 0x5571d6d21c60 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_file_opening_threads: 16 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.statistics: (nil) 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.use_fsync: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_log_file_size: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.keep_log_file_num: 1000 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.recycle_log_file_num: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.allow_fallocate: 1 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.allow_mmap_reads: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.allow_mmap_writes: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.use_direct_reads: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-25T04:22:56.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.create_missing_column_families: 0 2024-04-25T04:22:56.654 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.db_log_dir: 2024-04-25T04:22:56.654 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.wal_dir: 2024-04-25T04:22:56.654 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-25T04:22:56.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-25T04:22:56.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-25T04:22:56.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-25T04:22:56.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-25T04:22:56.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-25T04:22:56.655 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.advise_random_on_open: 1 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.db_write_buffer_size: 0 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.write_buffer_manager: 0x5571d6cf30e0 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.rate_limiter: (nil) 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.wal_recovery_mode: 2 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.enable_thread_tracking: 0 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.enable_pipelined_write: 0 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.unordered_write: 0 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-25T04:22:56.656 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.row_cache: None 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.wal_filter: None 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.allow_ingest_behind: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.two_write_queues: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.manual_wal_flush: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.wal_compression: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.atomic_flush: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.log_readahead_size: 0 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-25T04:22:56.657 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.best_efforts_recovery: 0 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.allow_data_in_errors: 0 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.db_host_id: __hostname__ 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_background_jobs: 2 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_background_compactions: -1 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_subcompactions: 1 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_total_wal_size: 0 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-25T04:22:56.658 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_open_files: -1 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bytes_per_sync: 0 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_readahead_size: 0 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_background_flushes: -1 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Compression algorithms supported: 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kZSTD supported: 0 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kXpressCompression supported: 0 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kLZ4HCCompression supported: 1 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kZlibCompression supported: 1 2024-04-25T04:22:56.659 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kSnappyCompression supported: 1 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kLZ4Compression supported: 1 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: kBZip2Compression supported: 0 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi112/store.db/MANIFEST-000009 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.merge_operator: 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_filter: None 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_filter_factory: None 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.sst_partitioner_factory: None 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5571d6d21e20) 2024-04-25T04:22:56.660 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: cache_index_and_filter_blocks: 1 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: pin_top_level_index_and_filter: 1 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_type: 0 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_block_index_type: 0 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_shortening: 1 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: checksum: 4 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: no_block_cache: 0 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache: 0x5571d6d251f0 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_name: BinnedLRUCache 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_options: 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: capacity : 536870912 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: num_shard_bits : 4 2024-04-25T04:22:56.661 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: strict_capacity_limit : 0 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: high_pri_pool_ratio: 0.000 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_cache_compressed: (nil) 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: persistent_cache: (nil) 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_size: 4096 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_size_deviation: 10 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_restart_interval: 16 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: index_block_restart_interval: 1 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: metadata_block_size: 4096 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: partition_filters: 0 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: use_delta_encoding: 1 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: filter_policy: bloomfilter 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: whole_key_filtering: 1 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: verify_compression: 0 2024-04-25T04:22:56.662 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: read_amp_bytes_per_bit: 0 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: format_version: 5 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: enable_index_compression: 1 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: block_align: 0 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_auto_readahead_size: 262144 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: prepopulate_block_cache: 0 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: initial_auto_readahead_size: 8192 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: num_file_reads_for_auto_readahead: 2 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.write_buffer_size: 33554432 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_write_buffer_number: 2 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression: NoCompression 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression: Disabled 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.prefix_extractor: nullptr 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-25T04:22:56.663 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.num_levels: 7 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.level: 32767 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.strategy: 0 2024-04-25T04:22:56.664 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.enabled: false 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.target_file_size_base: 67108864 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-25T04:22:56.665 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.arena_block_size: 1048576 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.disable_auto_compactions: 0 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-25T04:22:56.666 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.inplace_update_support: 0 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-25T04:22:56.667 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.bloom_locality: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.max_successive_merges: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.paranoid_file_checks: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.force_consistency_checks: 1 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.report_bg_io_stats: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.ttl: 2592000 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.enable_blob_files: false 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.min_blob_size: 0 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.blob_file_size: 268435456 2024-04-25T04:22:56.668 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.blob_file_starting_level: 0 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi112/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12249, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3dbe2906-3650-4378-bd2a-00ecd152e8c9 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018976377695, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018976389901, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3470254, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12254, "largest_seqno": 13704, "table_properties": {"data_size": 3463656, "index_size": 4081, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13319, "raw_average_key_size": 23, "raw_value_size": 3451023, "raw_average_value_size": 6012, "num_data_blocks": 187, "num_entries": 574, "num_filter_entries": 574, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714018976, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3dbe2906-3650-4378-bd2a-00ecd152e8c9", "db_session_id": "718BOIJCDRJG2KDGNXVX", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-04-25T04:22:56.669 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714018976390020, "job": 1, "event": "recovery_finished"} 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi112/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5571d6e42000 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: DB pointer 0x5571d6e2c000 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** DB Stats ** 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-25T04:22:56.670 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: L0 1/0 3.31 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 695.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: L6 1/0 7.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Sum 2/0 10.51 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 695.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 695.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** Compaction Stats [default] ** 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 695.6 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-04-25T04:22:56.671 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Cumulative compaction: 0.00 GB write, 195.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Interval compaction: 0.00 GB write, 195.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Block cache BinnedLRUCache@0x5571d6d251f0#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-04-25T04:22:56.672 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: starting mon.smithi112 rank 1 at public addrs [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] at bind addrs [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi112 fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???) e2 preinit fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???).mds e7 new map 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???).mds e7 print_map 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: e7 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: legacy client fscid: 1 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Filesystem 'foofs' (1) 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: fs_name foofs 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: epoch 5 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: created 2024-04-25T04:17:37.837625+0000 2024-04-25T04:22:56.673 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: modified 2024-04-25T04:17:43.857514+0000 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: tableserver 0 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: root 0 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: session_timeout 60 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: session_autoclose 300 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_file_size 1099511627776 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_xattr_size 65536 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: required_client_features {} 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: last_failure 0 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: last_failure_osd_epoch 0 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: max_mds 1 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: in 0 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: up {0=14498} 2024-04-25T04:22:56.674 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: failed 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: damaged 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: stopped 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: data_pools [3] 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: metadata_pool 2 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: inline_data disabled 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: balancer 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: bal_rank_mask -1 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: standby_count_wanted 1 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: qdb_cluster leader: 0 members: 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: [mds.foofs.smithi078.nlrtle{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/131251842,v1:172.21.15.78:6835/131251842] compat {c=[1],r=[1],i=[7ff]}] 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: Standby daemons: 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: 2024-04-25T04:22:56.675 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout: [mds.foofs.smithi112.xminls{-1:24271} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] compat {c=[1],r=[1],i=[7ff]}] 2024-04-25T04:22:56.676 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-25T04:22:56.676 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-25T04:22:56.676 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-25T04:22:56.676 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-25T04:22:56.676 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 ceph-mon[50007]: mon.smithi112@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-04-25T04:22:56.676 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 bash[49989]: cf109ce58bbce47b69b86cf9f1123581e047845b354ccae8611cd9f3e9baa214 2024-04-25T04:22:56.676 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:56 smithi112 systemd[1]: Started Ceph mon.smithi112 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:22:56.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 140079 11.24 MB/sec execute 160 sec latency 43.217 ms 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: mon.smithi112 calling monitor election 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: mon.smithi078 calling monitor election 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: mon.smithi078 is new leader, mons smithi078,smithi112 in quorum (ranks 0,1) 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: monmap epoch 3 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: last_changed 2024-04-25T04:22:56.484110+0000 2024-04-25T04:22:57.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: created 2024-04-25T04:10:09.587319+0000 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: min_mon_release 19 (squid) 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: election_strategy: 1 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: 1: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: osdmap e75: 8 total, 8 up, 8 in 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: mgrmap e39: smithi078.eugwik(active, since 18s), standbys: smithi112.lvtlnw 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: overall HEALTH_OK 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:57.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:57.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 140700 11.30 MB/sec execute 161 sec latency 24.493 ms 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mon metadata", "id": "smithi112"}]: dispatch 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: mon.smithi112 calling monitor election 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: mon.smithi078 calling monitor election 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: mon.smithi078 is new leader, mons smithi078,smithi112 in quorum (ranks 0,1) 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: monmap epoch 3 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: last_changed 2024-04-25T04:22:56.484110+0000 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: created 2024-04-25T04:10:09.587319+0000 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: min_mon_release 19 (squid) 2024-04-25T04:22:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: election_strategy: 1 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: 1: [v2:172.21.15.112:3300/0,v1:172.21.15.112:6789/0] mon.smithi112 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: osdmap e75: 8 total, 8 up, 8 in 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: mgrmap e39: smithi078.eugwik(active, since 18s), standbys: smithi112.lvtlnw 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: overall HEALTH_OK 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:22:57.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:22:58.530 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:22:58 smithi078 ceph-mon[62942]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-04-25T04:22:58.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141358 11.37 MB/sec execute 162 sec latency 33.692 ms 2024-04-25T04:22:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:22:58 smithi112 ceph-mon[50007]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 412 op/s 2024-04-25T04:22:59.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141861 11.36 MB/sec execute 163 sec latency 45.605 ms 2024-04-25T04:23:00.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:00 smithi078 ceph-mon[62942]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.3 MiB/s wr, 726 op/s 2024-04-25T04:23:00.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:00 smithi112 ceph-mon[50007]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.3 MiB/s wr, 726 op/s 2024-04-25T04:23:00.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 142418 11.38 MB/sec execute 164 sec latency 56.130 ms 2024-04-25T04:23:01.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:01 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:01.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:01 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 142890 11.39 MB/sec execute 165 sec latency 29.013 ms 2024-04-25T04:23:01.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:01 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:01.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:01 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:02.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143490 11.39 MB/sec execute 166 sec latency 33.780 ms 2024-04-25T04:23:02.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:02.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:02.841 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:02 smithi112 ceph-mon[50007]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 525 op/s 2024-04-25T04:23:02.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:02.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:02.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:02 smithi078 ceph-mon[62942]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 525 op/s 2024-04-25T04:23:03.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144104 11.44 MB/sec execute 167 sec latency 48.112 ms 2024-04-25T04:23:04.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:04 smithi078 ceph-mon[62942]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 525 op/s 2024-04-25T04:23:04.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144662 11.46 MB/sec execute 168 sec latency 32.460 ms 2024-04-25T04:23:04.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:04 smithi112 ceph-mon[50007]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 525 op/s 2024-04-25T04:23:05.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145174 11.51 MB/sec execute 169 sec latency 30.265 ms 2024-04-25T04:23:06.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145624 11.48 MB/sec execute 170 sec latency 32.042 ms 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: pgmap v17: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:23:07.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:07.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: pgmap v17: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:23:07.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:07.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146033 11.50 MB/sec execute 171 sec latency 41.243 ms 2024-04-25T04:23:08.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:07 smithi078 ceph-mon[62942]: Reconfiguring mon.smithi078 (monmap changed)... 2024-04-25T04:23:08.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:07 smithi078 ceph-mon[62942]: Reconfiguring daemon mon.smithi078 on smithi078 2024-04-25T04:23:08.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:07 smithi112 ceph-mon[50007]: Reconfiguring mon.smithi078 (monmap changed)... 2024-04-25T04:23:08.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:07 smithi112 ceph-mon[50007]: Reconfiguring daemon mon.smithi078 on smithi078 2024-04-25T04:23:08.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146551 11.50 MB/sec execute 172 sec latency 26.591 ms 2024-04-25T04:23:09.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:08 smithi078 ceph-mon[62942]: pgmap v18: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 647 op/s 2024-04-25T04:23:09.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:09.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:08 smithi112 ceph-mon[50007]: pgmap v18: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 647 op/s 2024-04-25T04:23:09.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:09.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147041 11.48 MB/sec execute 173 sec latency 42.235 ms 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: Reconfiguring mgr.smithi078.eugwik (monmap changed)... 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: Reconfiguring daemon mgr.smithi078.eugwik on smithi078 2024-04-25T04:23:10.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:10 smithi078 ceph-mon[62942]: pgmap v19: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 947 op/s 2024-04-25T04:23:10.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: Reconfiguring mgr.smithi078.eugwik (monmap changed)... 2024-04-25T04:23:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.eugwik", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:23:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:23:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: Reconfiguring daemon mgr.smithi078.eugwik on smithi078 2024-04-25T04:23:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:10 smithi112 ceph-mon[50007]: pgmap v19: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 947 op/s 2024-04-25T04:23:10.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147471 11.49 MB/sec execute 174 sec latency 36.942 ms 2024-04-25T04:23:11.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147826 11.49 MB/sec execute 175 sec latency 48.036 ms 2024-04-25T04:23:12.344 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:23:12.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:12.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:12.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:12 smithi078 ceph-mon[62942]: Reconfiguring crash.smithi078 (monmap changed)... 2024-04-25T04:23:12.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:12.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:12.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:12 smithi078 ceph-mon[62942]: Reconfiguring daemon crash.smithi078 on smithi078 2024-04-25T04:23:12.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:12 smithi078 ceph-mon[62942]: pgmap v20: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 633 op/s 2024-04-25T04:23:12.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148157 11.48 MB/sec execute 176 sec latency 64.052 ms 2024-04-25T04:23:12.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:12.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:12.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:12 smithi112 ceph-mon[50007]: Reconfiguring crash.smithi078 (monmap changed)... 2024-04-25T04:23:12.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:12.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:12.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:12 smithi112 ceph-mon[50007]: Reconfiguring daemon crash.smithi078 on smithi078 2024-04-25T04:23:12.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:12 smithi112 ceph-mon[50007]: pgmap v20: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 633 op/s 2024-04-25T04:23:13.396 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:23:13.396 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (2m) 25s ago 11m 26.1M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:23:13.396 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (11m) 25s ago 11m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:23:13.396 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (10m) 12s ago 10m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:23:13.396 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 25s ago 11m 45.6M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:23:13.396 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (5m) 25s ago 5m 40.1M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (5m) 12s ago 5m 14.2M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (118s) 25s ago 12m 480M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (65s) 12s ago 10m 449M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (45s) 25s ago 13m 31.0M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (17s) 12s ago 10m 22.6M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (5m) 25s ago 5m 274M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (5m) 12s ago 5m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 25s ago 11m 13.2M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (2m) 12s ago 10m 18.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 25s ago 9m 471M 1936M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 25s ago 9m 406M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 25s ago 8m 289M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:23:13.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (8m) 25s ago 8m 444M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:23:13.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (8m) 12s ago 8m 472M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:23:13.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (7m) 12s ago 7m 550M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:23:13.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (7m) 12s ago 7m 456M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:23:13.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (6m) 12s ago 6m 364M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:23:13.398 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (72s) 25s ago 10m 56.6M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:23:13.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:13 smithi078 ceph-mon[62942]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:13.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:13 smithi078 ceph-mon[62942]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:13.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148478 11.49 MB/sec execute 177 sec latency 39.496 ms 2024-04-25T04:23:13.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:13 smithi112 ceph-mon[50007]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:13.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:13 smithi112 ceph-mon[50007]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:23:13.932 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:23:13.933 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:13.933 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:23:13.933 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-25T04:23:13.933 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 4 2024-04-25T04:23:13.933 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:23:13.933 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:23:14.503 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:23:14.503 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "4/23 daemons upgraded", 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:23:14.504 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:23:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:14 smithi078 ceph-mon[62942]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:14 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3640915020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:14 smithi078 ceph-mon[62942]: pgmap v21: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 633 op/s 2024-04-25T04:23:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:14 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:14 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:14 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T04:23:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:14 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:14.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148864 11.47 MB/sec execute 178 sec latency 30.650 ms 2024-04-25T04:23:14.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:14 smithi112 ceph-mon[50007]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:14.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:14 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3640915020' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:14.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:14 smithi112 ceph-mon[50007]: pgmap v21: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 633 op/s 2024-04-25T04:23:14.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:14 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:14.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:14 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:14.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:14 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T04:23:14.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:14 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:15.168 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:23:15.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:15 smithi078 ceph-mon[62942]: Reconfiguring osd.0 (monmap changed)... 2024-04-25T04:23:15.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:15 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.0 on smithi078 2024-04-25T04:23:15.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:15 smithi078 ceph-mon[62942]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:15.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:15 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/172196812' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:23:15.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149312 11.46 MB/sec execute 179 sec latency 51.122 ms 2024-04-25T04:23:16.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:15 smithi112 ceph-mon[50007]: Reconfiguring osd.0 (monmap changed)... 2024-04-25T04:23:16.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:15 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.0 on smithi078 2024-04-25T04:23:16.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:15 smithi112 ceph-mon[50007]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:16.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:15 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/172196812' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:23:16.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:16 smithi078 ceph-mon[62942]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 863 op/s 2024-04-25T04:23:16.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149730 11.47 MB/sec execute 180 sec latency 32.149 ms 2024-04-25T04:23:17.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:16 smithi112 ceph-mon[50007]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 863 op/s 2024-04-25T04:23:17.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150217 11.48 MB/sec execute 181 sec latency 41.193 ms 2024-04-25T04:23:18.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:18 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:18.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:18 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:18.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:18 smithi078 ceph-mon[62942]: Reconfiguring osd.1 (monmap changed)... 2024-04-25T04:23:18.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:18 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T04:23:18.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:18 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:18.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:18 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.1 on smithi078 2024-04-25T04:23:18.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:18 smithi078 ceph-mon[62942]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-04-25T04:23:18.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150637 11.45 MB/sec execute 182 sec latency 43.651 ms 2024-04-25T04:23:18.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:18 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:18.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:18 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:18.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:18 smithi112 ceph-mon[50007]: Reconfiguring osd.1 (monmap changed)... 2024-04-25T04:23:18.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:18 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T04:23:18.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:18 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:18.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:18 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.1 on smithi078 2024-04-25T04:23:18.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:18 smithi112 ceph-mon[50007]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-04-25T04:23:19.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151002 11.45 MB/sec execute 183 sec latency 37.165 ms 2024-04-25T04:23:20.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:20 smithi078 ceph-mon[62942]: pgmap v24: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 794 op/s 2024-04-25T04:23:20.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:20 smithi112 ceph-mon[50007]: pgmap v24: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 794 op/s 2024-04-25T04:23:20.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151451 11.46 MB/sec execute 184 sec latency 48.119 ms 2024-04-25T04:23:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:21 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:21 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:21 smithi078 ceph-mon[62942]: Reconfiguring osd.2 (monmap changed)... 2024-04-25T04:23:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:21 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T04:23:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:21 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:21 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.2 on smithi078 2024-04-25T04:23:21.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:21 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:21 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:21 smithi112 ceph-mon[50007]: Reconfiguring osd.2 (monmap changed)... 2024-04-25T04:23:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:21 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T04:23:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:21 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:21 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.2 on smithi078 2024-04-25T04:23:21.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151857 11.47 MB/sec execute 185 sec latency 39.154 ms 2024-04-25T04:23:22.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:22 smithi078 ceph-mon[62942]: pgmap v25: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 494 op/s 2024-04-25T04:23:22.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:22 smithi112 ceph-mon[50007]: pgmap v25: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 494 op/s 2024-04-25T04:23:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152157 11.48 MB/sec execute 186 sec latency 40.818 ms 2024-04-25T04:23:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:23.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:23.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:23.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:23.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152473 11.44 MB/sec execute 187 sec latency 51.526 ms 2024-04-25T04:23:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:24 smithi078 ceph-mon[62942]: Reconfiguring osd.3 (monmap changed)... 2024-04-25T04:23:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T04:23:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:24 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.3 on smithi078 2024-04-25T04:23:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:24 smithi078 ceph-mon[62942]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 494 op/s 2024-04-25T04:23:24.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:24 smithi112 ceph-mon[50007]: Reconfiguring osd.3 (monmap changed)... 2024-04-25T04:23:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T04:23:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:24 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.3 on smithi078 2024-04-25T04:23:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:24 smithi112 ceph-mon[50007]: pgmap v26: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 494 op/s 2024-04-25T04:23:24.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152812 11.42 MB/sec execute 188 sec latency 52.910 ms 2024-04-25T04:23:25.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153287 11.43 MB/sec execute 189 sec latency 34.603 ms 2024-04-25T04:23:26.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153949 11.45 MB/sec execute 190 sec latency 23.883 ms 2024-04-25T04:23:27.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:26 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:27.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:26 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:27.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:26 smithi078 ceph-mon[62942]: Reconfiguring mds.foofs.smithi078.nlrtle (monmap changed)... 2024-04-25T04:23:27.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:26 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:23:27.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:26 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:27.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:26 smithi078 ceph-mon[62942]: Reconfiguring daemon mds.foofs.smithi078.nlrtle on smithi078 2024-04-25T04:23:27.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:26 smithi078 ceph-mon[62942]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 709 op/s 2024-04-25T04:23:27.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:27.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:27.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:27 smithi112 ceph-mon[50007]: Reconfiguring mds.foofs.smithi078.nlrtle (monmap changed)... 2024-04-25T04:23:27.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:23:27.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:27.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:27 smithi112 ceph-mon[50007]: Reconfiguring daemon mds.foofs.smithi078.nlrtle on smithi078 2024-04-25T04:23:27.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:27 smithi112 ceph-mon[50007]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 709 op/s 2024-04-25T04:23:27.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154472 11.46 MB/sec execute 191 sec latency 29.989 ms 2024-04-25T04:23:28.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:28 smithi078 ceph-mon[62942]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-04-25T04:23:28.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:28 smithi112 ceph-mon[50007]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-04-25T04:23:28.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155021 11.48 MB/sec execute 192 sec latency 44.918 ms 2024-04-25T04:23:29.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:29.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:29.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:29 smithi078 ceph-mon[62942]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-25T04:23:29.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:29.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:29.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:29 smithi078 ceph-mon[62942]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-25T04:23:29.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155457 11.50 MB/sec execute 193 sec latency 43.430 ms 2024-04-25T04:23:30.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:30.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:30.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:29 smithi112 ceph-mon[50007]: Reconfiguring crash.smithi112 (monmap changed)... 2024-04-25T04:23:30.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:30.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:30.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:29 smithi112 ceph-mon[50007]: Reconfiguring daemon crash.smithi112 on smithi112 2024-04-25T04:23:30.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155849 11.51 MB/sec execute 194 sec latency 60.275 ms 2024-04-25T04:23:30.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:30 smithi078 ceph-mon[62942]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 793 op/s 2024-04-25T04:23:31.024 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:30 smithi112 ceph-mon[50007]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 793 op/s 2024-04-25T04:23:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156258 11.49 MB/sec execute 195 sec latency 32.208 ms 2024-04-25T04:23:32.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:32.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:32.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:32 smithi078 ceph-mon[62942]: Reconfiguring mgr.smithi112.lvtlnw (monmap changed)... 2024-04-25T04:23:32.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:23:32.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:23:32.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:32.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:32 smithi078 ceph-mon[62942]: Reconfiguring daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:23:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:32 smithi112 ceph-mon[50007]: Reconfiguring mgr.smithi112.lvtlnw (monmap changed)... 2024-04-25T04:23:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi112.lvtlnw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-25T04:23:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-25T04:23:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:32 smithi112 ceph-mon[50007]: Reconfiguring daemon mgr.smithi112.lvtlnw on smithi112 2024-04-25T04:23:32.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156680 11.49 MB/sec execute 196 sec latency 54.461 ms 2024-04-25T04:23:33.195 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:33 smithi078 ceph-mon[62942]: pgmap v30: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-04-25T04:23:33.195 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:33.195 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:33.195 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:23:33.195 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:23:33.195 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:33.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:33 smithi112 ceph-mon[50007]: pgmap v30: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-04-25T04:23:33.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:33.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:33.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-25T04:23:33.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-25T04:23:33.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157280 11.51 MB/sec execute 197 sec latency 53.890 ms 2024-04-25T04:23:34.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:34 smithi078 ceph-mon[62942]: Reconfiguring mon.smithi112 (monmap changed)... 2024-04-25T04:23:34.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:34 smithi078 ceph-mon[62942]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-25T04:23:34.334 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:34 smithi112 ceph-mon[50007]: Reconfiguring mon.smithi112 (monmap changed)... 2024-04-25T04:23:34.334 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:34 smithi112 ceph-mon[50007]: Reconfiguring daemon mon.smithi112 on smithi112 2024-04-25T04:23:34.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157694 11.49 MB/sec execute 198 sec latency 43.180 ms 2024-04-25T04:23:35.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:35 smithi078 ceph-mon[62942]: pgmap v31: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-04-25T04:23:35.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:35.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:35.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T04:23:35.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:35.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:35 smithi112 ceph-mon[50007]: pgmap v31: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 527 op/s 2024-04-25T04:23:35.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T04:23:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:35.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158035 11.48 MB/sec execute 199 sec latency 49.881 ms 2024-04-25T04:23:36.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:36 smithi112 ceph-mon[50007]: Reconfiguring osd.4 (monmap changed)... 2024-04-25T04:23:36.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:36 smithi078 ceph-mon[62942]: Reconfiguring osd.4 (monmap changed)... 2024-04-25T04:23:36.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:36 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.4 on smithi112 2024-04-25T04:23:36.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:36 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.4 on smithi112 2024-04-25T04:23:36.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158420 11.47 MB/sec execute 200 sec latency 124.398 ms 2024-04-25T04:23:37.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:37 smithi078 ceph-mon[62942]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 803 op/s 2024-04-25T04:23:37.239 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:37 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:37.239 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:37 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:37.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:37 smithi112 ceph-mon[50007]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 803 op/s 2024-04-25T04:23:37.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:37 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:37.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:37 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:37.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159009 11.52 MB/sec execute 201 sec latency 47.605 ms 2024-04-25T04:23:38.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:38 smithi078 ceph-mon[62942]: Reconfiguring osd.5 (monmap changed)... 2024-04-25T04:23:38.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T04:23:38.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:38.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:38 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.5 on smithi112 2024-04-25T04:23:38.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:38 smithi112 ceph-mon[50007]: Reconfiguring osd.5 (monmap changed)... 2024-04-25T04:23:38.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T04:23:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:38.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:38 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.5 on smithi112 2024-04-25T04:23:38.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159617 11.54 MB/sec execute 202 sec latency 38.967 ms 2024-04-25T04:23:39.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:39 smithi078 ceph-mon[62942]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-04-25T04:23:39.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:39.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:39.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:39.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T04:23:39.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:39.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:39 smithi112 ceph-mon[50007]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-04-25T04:23:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T04:23:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:39.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160101 11.53 MB/sec execute 203 sec latency 22.370 ms 2024-04-25T04:23:40.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:40 smithi078 ceph-mon[62942]: Reconfiguring osd.6 (monmap changed)... 2024-04-25T04:23:40.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:40 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.6 on smithi112 2024-04-25T04:23:40.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:40 smithi112 ceph-mon[50007]: Reconfiguring osd.6 (monmap changed)... 2024-04-25T04:23:40.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:40 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.6 on smithi112 2024-04-25T04:23:40.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160708 11.56 MB/sec execute 204 sec latency 30.047 ms 2024-04-25T04:23:41.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:41 smithi078 ceph-mon[62942]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 869 op/s 2024-04-25T04:23:41.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:41 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:41.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:41 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:41.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:41 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T04:23:41.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:41 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:41.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:41 smithi112 ceph-mon[50007]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 869 op/s 2024-04-25T04:23:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:41 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:41 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:41 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T04:23:41.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:41 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:41.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161270 11.56 MB/sec execute 205 sec latency 58.790 ms 2024-04-25T04:23:42.233 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:42 smithi112 ceph-mon[50007]: Reconfiguring osd.7 (monmap changed)... 2024-04-25T04:23:42.233 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:42 smithi112 ceph-mon[50007]: Reconfiguring daemon osd.7 on smithi112 2024-04-25T04:23:42.281 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:42 smithi078 ceph-mon[62942]: Reconfiguring osd.7 (monmap changed)... 2024-04-25T04:23:42.282 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:42 smithi078 ceph-mon[62942]: Reconfiguring daemon osd.7 on smithi112 2024-04-25T04:23:42.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161775 11.58 MB/sec execute 206 sec latency 40.784 ms 2024-04-25T04:23:43.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:43 smithi078 ceph-mon[62942]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-04-25T04:23:43.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:43.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:43.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:23:43.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:43.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:43 smithi112 ceph-mon[50007]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-04-25T04:23:43.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:43.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:43.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:23:43.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:43.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162279 11.60 MB/sec execute 207 sec latency 34.286 ms 2024-04-25T04:23:44.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:44 smithi078 ceph-mon[62942]: Reconfiguring mds.foofs.smithi112.xminls (monmap changed)... 2024-04-25T04:23:44.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:44 smithi078 ceph-mon[62942]: Reconfiguring daemon mds.foofs.smithi112.xminls on smithi112 2024-04-25T04:23:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:44 smithi112 ceph-mon[50007]: Reconfiguring mds.foofs.smithi112.xminls (monmap changed)... 2024-04-25T04:23:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:44 smithi112 ceph-mon[50007]: Reconfiguring daemon mds.foofs.smithi112.xminls on smithi112 2024-04-25T04:23:44.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162652 11.62 MB/sec execute 208 sec latency 45.958 ms 2024-04-25T04:23:45.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: pgmap v36: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-04-25T04:23:45.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:45.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:45.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:23:45.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:45.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:45.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all mon 2024-04-25T04:23:45.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:45.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]: dispatch 2024-04-25T04:23:45.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-04-25T04:23:45.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]: dispatch 2024-04-25T04:23:45.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]': finished 2024-04-25T04:23:45.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: pgmap v36: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all mon 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]: dispatch 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]: dispatch 2024-04-25T04:23:45.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi112"}]': finished 2024-04-25T04:23:45.739 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:23:45.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162903 11.60 MB/sec execute 209 sec latency 57.854 ms 2024-04-25T04:23:46.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:46 smithi078 ceph-mon[62942]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:46.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:46 smithi078 ceph-mon[62942]: pgmap v37: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 877 op/s 2024-04-25T04:23:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:46 smithi112 ceph-mon[50007]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:46 smithi112 ceph-mon[50007]: pgmap v37: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 877 op/s 2024-04-25T04:23:46.717 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:23:46.717 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 58s ago 12m 26.1M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:23:46.717 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (12m) 58s ago 12m 6400k - 17.2.0 e1d6a67b021e 60b8b55465d1 2024-04-25T04:23:46.717 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (11m) 46s ago 11m 6413k - 17.2.0 e1d6a67b021e bce0f4b48392 2024-04-25T04:23:46.717 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 58s ago 11m 45.6M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (6m) 58s ago 6m 40.1M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (6m) 46s ago 6m 14.2M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (2m) 58s ago 13m 480M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (98s) 46s ago 11m 449M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (78s) 58s ago 13m 31.0M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (50s) 46s ago 11m 22.6M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (5m) 58s ago 5m 274M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (5m) 46s ago 5m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 58s ago 11m 13.2M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (2m) 46s ago 11m 18.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (10m) 58s ago 10m 471M 1936M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 58s ago 9m 406M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:23:46.718 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (9m) 58s ago 9m 289M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:23:46.719 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 58s ago 9m 444M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:23:46.719 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (8m) 46s ago 8m 472M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:23:46.719 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (8m) 46s ago 8m 550M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:23:46.719 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (7m) 46s ago 7m 456M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:23:46.719 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (7m) 46s ago 7m 364M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:23:46.719 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (105s) 58s ago 10m 56.6M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:23:46.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163254 11.57 MB/sec execute 210 sec latency 65.488 ms 2024-04-25T04:23:47.211 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:23:47.211 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:23:47.211 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:23:47.211 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 4 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:23:47.212 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:23:47.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:47 smithi078 ceph-mon[62942]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:47.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:47 smithi078 ceph-mon[62942]: from='client.34186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:47.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:47 smithi112 ceph-mon[50007]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:47.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:47 smithi112 ceph-mon[50007]: from='client.34186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:47.657 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:23:47.657 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:23:47.657 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:23:47.657 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:23:47.657 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:23:47.657 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:23:47.658 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-04-25T04:23:47.658 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:23:47.658 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "4/23 daemons upgraded", 2024-04-25T04:23:47.658 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading crash daemons", 2024-04-25T04:23:47.658 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:23:47.658 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:23:47.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163494 11.54 MB/sec execute 211 sec latency 78.855 ms 2024-04-25T04:23:48.286 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1309584713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: Upgrade: Updating crash.smithi078 (1/2) 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: Deploying daemon crash.smithi078 on smithi078 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:48.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:48 smithi078 ceph-mon[62942]: pgmap v38: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.3 MiB/s wr, 602 op/s 2024-04-25T04:23:48.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1309584713' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:23:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: Upgrade: Updating crash.smithi078 (1/2) 2024-04-25T04:23:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: Deploying daemon crash.smithi078 on smithi078 2024-04-25T04:23:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:23:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:48 smithi112 ceph-mon[50007]: pgmap v38: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.3 MiB/s wr, 602 op/s 2024-04-25T04:23:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163762 11.52 MB/sec execute 212 sec latency 44.470 ms 2024-04-25T04:23:49.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:49 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3916221415' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:23:49.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:49 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3916221415' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:23:49.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164134 11.51 MB/sec execute 213 sec latency 46.721 ms 2024-04-25T04:23:50.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:50 smithi078 ceph-mon[62942]: pgmap v39: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 792 op/s 2024-04-25T04:23:50.596 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:50 smithi112 ceph-mon[50007]: pgmap v39: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 792 op/s 2024-04-25T04:23:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164649 11.51 MB/sec execute 214 sec latency 36.094 ms 2024-04-25T04:23:51.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165109 11.52 MB/sec execute 215 sec latency 45.512 ms 2024-04-25T04:23:52.205 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:52 smithi078 ceph-mon[62942]: pgmap v40: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-04-25T04:23:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:52 smithi112 ceph-mon[50007]: pgmap v40: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-04-25T04:23:52.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165674 11.53 MB/sec execute 216 sec latency 39.652 ms 2024-04-25T04:23:53.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:23:53.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166292 11.60 MB/sec execute 217 sec latency 35.031 ms 2024-04-25T04:23:54.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:54.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:54.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:54 smithi078 ceph-mon[62942]: pgmap v41: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-04-25T04:23:54.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166741 11.57 MB/sec execute 218 sec latency 39.030 ms 2024-04-25T04:23:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:54 smithi112 ceph-mon[50007]: pgmap v41: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-04-25T04:23:55.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167138 11.56 MB/sec execute 219 sec latency 39.642 ms 2024-04-25T04:23:56.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:56 smithi078 ceph-mon[62942]: Upgrade: Updating crash.smithi112 (2/2) 2024-04-25T04:23:56.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:56 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:56.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:56 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:56.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:56 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:56.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:56 smithi078 ceph-mon[62942]: Deploying daemon crash.smithi112 on smithi112 2024-04-25T04:23:56.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:56 smithi078 ceph-mon[62942]: pgmap v42: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 812 op/s 2024-04-25T04:23:56.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:56 smithi112 ceph-mon[50007]: Upgrade: Updating crash.smithi112 (2/2) 2024-04-25T04:23:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:56 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:23:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:56 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi112", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-25T04:23:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:56 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:23:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:56 smithi112 ceph-mon[50007]: Deploying daemon crash.smithi112 on smithi112 2024-04-25T04:23:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:56 smithi112 ceph-mon[50007]: pgmap v42: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 812 op/s 2024-04-25T04:23:56.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167511 11.56 MB/sec execute 220 sec latency 56.608 ms 2024-04-25T04:23:57.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167873 11.55 MB/sec execute 221 sec latency 86.302 ms 2024-04-25T04:23:58.228 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:23:58 smithi078 ceph-mon[62942]: pgmap v43: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 491 op/s 2024-04-25T04:23:58.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:23:58 smithi112 ceph-mon[50007]: pgmap v43: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 491 op/s 2024-04-25T04:23:58.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168199 11.52 MB/sec execute 222 sec latency 60.352 ms 2024-04-25T04:23:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168492 11.50 MB/sec execute 223 sec latency 44.082 ms 2024-04-25T04:24:00.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:00 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:00.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:00 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:00.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:00 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:00.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:00 smithi078 ceph-mon[62942]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 727 op/s 2024-04-25T04:24:00.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168764 11.49 MB/sec execute 224 sec latency 47.426 ms 2024-04-25T04:24:00.976 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:00 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:00.976 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:00 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:00.976 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:00 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:00.976 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:00 smithi112 ceph-mon[50007]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 727 op/s 2024-04-25T04:24:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169190 11.49 MB/sec execute 225 sec latency 36.150 ms 2024-04-25T04:24:02.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:02 smithi078 ceph-mon[62942]: pgmap v45: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 536 op/s 2024-04-25T04:24:02.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:02 smithi112 ceph-mon[50007]: pgmap v45: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 536 op/s 2024-04-25T04:24:02.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169531 11.50 MB/sec execute 226 sec latency 42.013 ms 2024-04-25T04:24:03.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169832 11.50 MB/sec execute 227 sec latency 47.569 ms 2024-04-25T04:24:04.329 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:04 smithi078 ceph-mon[62942]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 538 op/s 2024-04-25T04:24:04.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:04 smithi112 ceph-mon[50007]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 538 op/s 2024-04-25T04:24:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170057 11.47 MB/sec execute 228 sec latency 61.084 ms 2024-04-25T04:24:05.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:05 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:05.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:05 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:05 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:05 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:05.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170343 11.45 MB/sec execute 229 sec latency 64.567 ms 2024-04-25T04:24:06.694 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:06.694 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:06.694 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:06 smithi112 ceph-mon[50007]: pgmap v47: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 731 op/s 2024-04-25T04:24:06.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170580 11.41 MB/sec execute 230 sec latency 57.170 ms 2024-04-25T04:24:06.905 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:06.906 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:06.906 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:06 smithi078 ceph-mon[62942]: pgmap v47: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 731 op/s 2024-04-25T04:24:07.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170772 11.38 MB/sec execute 231 sec latency 110.181 ms 2024-04-25T04:24:08.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:08 smithi078 ceph-mon[62942]: pgmap v48: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-04-25T04:24:08.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:08 smithi112 ceph-mon[50007]: pgmap v48: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-04-25T04:24:08.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:08.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171059 11.37 MB/sec execute 232 sec latency 47.118 ms 2024-04-25T04:24:09.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171455 11.36 MB/sec execute 233 sec latency 64.320 ms 2024-04-25T04:24:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:10 smithi078 ceph-mon[62942]: pgmap v49: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 596 op/s 2024-04-25T04:24:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171768 11.34 MB/sec execute 234 sec latency 57.674 ms 2024-04-25T04:24:10.897 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.897 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.897 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.897 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:10.897 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:10 smithi112 ceph-mon[50007]: pgmap v49: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 596 op/s 2024-04-25T04:24:11.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:11 smithi078 ceph-mon[62942]: mgrmap e40: smithi078.eugwik(active, since 92s), standbys: smithi112.lvtlnw 2024-04-25T04:24:11.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:11 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:11.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:11 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:11.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172033 11.31 MB/sec execute 235 sec latency 57.270 ms 2024-04-25T04:24:12.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:11 smithi112 ceph-mon[50007]: mgrmap e40: smithi078.eugwik(active, since 92s), standbys: smithi112.lvtlnw 2024-04-25T04:24:12.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:11 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:12.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:11 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:12.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172271 11.31 MB/sec execute 236 sec latency 75.590 ms 2024-04-25T04:24:12.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:12 smithi078 ceph-mon[62942]: pgmap v50: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 360 op/s 2024-04-25T04:24:13.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:12 smithi112 ceph-mon[50007]: pgmap v50: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 360 op/s 2024-04-25T04:24:13.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172591 11.29 MB/sec execute 237 sec latency 51.332 ms 2024-04-25T04:24:14.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:14 smithi078 ceph-mon[62942]: pgmap v51: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.8 MiB/s wr, 362 op/s 2024-04-25T04:24:14.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:14 smithi112 ceph-mon[50007]: pgmap v51: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.8 MiB/s wr, 362 op/s 2024-04-25T04:24:14.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172903 11.30 MB/sec execute 238 sec latency 50.799 ms 2024-04-25T04:24:15.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173161 11.28 MB/sec execute 239 sec latency 64.346 ms 2024-04-25T04:24:16.208 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:16 smithi078 ceph-mon[62942]: pgmap v52: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.5 MiB/s wr, 539 op/s 2024-04-25T04:24:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:16 smithi112 ceph-mon[50007]: pgmap v52: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.5 MiB/s wr, 539 op/s 2024-04-25T04:24:16.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173528 11.29 MB/sec execute 240 sec latency 35.927 ms 2024-04-25T04:24:17.472 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all crash 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]: dispatch 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-04-25T04:24:17.473 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]: dispatch 2024-04-25T04:24:17.474 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]': finished 2024-04-25T04:24:17.474 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-25T04:24:17.474 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-25T04:24:17.474 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:17 smithi078 ceph-mon[62942]: Upgrade: osd.0 is safe to restart 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all crash 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]: dispatch 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]: dispatch 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi112"}]': finished 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-25T04:24:17.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:17 smithi112 ceph-mon[50007]: Upgrade: osd.0 is safe to restart 2024-04-25T04:24:17.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173929 11.27 MB/sec execute 241 sec latency 57.736 ms 2024-04-25T04:24:18.480 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:18 smithi078 ceph-mon[62942]: pgmap v53: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.2 MiB/s wr, 345 op/s 2024-04-25T04:24:18.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:18 smithi112 ceph-mon[50007]: pgmap v53: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.2 MiB/s wr, 345 op/s 2024-04-25T04:24:18.823 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:24:18.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174188 11.25 MB/sec execute 242 sec latency 41.976 ms 2024-04-25T04:24:19.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:19 smithi078 ceph-mon[62942]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:19.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:19 smithi112 ceph-mon[50007]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:19.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174393 11.23 MB/sec execute 243 sec latency 65.731 ms 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 10s ago 12m 25.7M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (27s) 10s ago 12m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (20s) 15s ago 12m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (3m) 10s ago 12m 55.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (6m) 10s ago 6m 40.2M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (6m) 15s ago 6m 14.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (3m) 10s ago 13m 499M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (2m) 15s ago 11m 450M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (112s) 10s ago 14m 42.7M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:24:19.851 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (83s) 15s ago 11m 35.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (6m) 10s ago 6m 276M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (6m) 15s ago 6m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 10s ago 12m 19.9M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (3m) 15s ago 11m 16.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (11m) 10s ago 11m 318M 1936M 17.2.0 e1d6a67b021e 21fbb2803873 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (10m) 10s ago 10m 497M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 10s ago 10m 389M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 10s ago 9m 517M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (9m) 15s ago 9m 528M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (8m) 15s ago 8m 381M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (8m) 15s ago 8m 556M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (8m) 15s ago 8m 466M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:24:19.852 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (2m) 10s ago 11m 62.8M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:24:20.507 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:20.508 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:24:20.509 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-25T04:24:20.509 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 4 2024-04-25T04:24:20.509 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:24:20.509 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: Upgrade: Updating osd.0 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: Deploying daemon osd.0 on smithi078 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:20.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:20 smithi078 ceph-mon[62942]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 541 op/s 2024-04-25T04:24:20.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174651 11.21 MB/sec execute 244 sec latency 61.155 ms 2024-04-25T04:24:20.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:20.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: Upgrade: Updating osd.0 2024-04-25T04:24:20.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:20.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-25T04:24:20.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:24:20.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: Deploying daemon osd.0 on smithi078 2024-04-25T04:24:20.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:20.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:20 smithi112 ceph-mon[50007]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 541 op/s 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "6/23 daemons upgraded", 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:24:21.020 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:24:21.529 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:21 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1056876276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:21.529 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:21 smithi078 ceph-mon[62942]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:21.733 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:24:21.824 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:21 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1056876276' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:21.825 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:21 smithi112 ceph-mon[50007]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:21.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175009 11.20 MB/sec execute 245 sec latency 62.359 ms 2024-04-25T04:24:22.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:22 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3462304614' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:24:22.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:22 smithi078 ceph-mon[62942]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 375 op/s 2024-04-25T04:24:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175341 11.18 MB/sec execute 246 sec latency 63.235 ms 2024-04-25T04:24:22.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:22 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3462304614' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:24:22.846 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:22 smithi112 ceph-mon[50007]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 375 op/s 2024-04-25T04:24:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:23.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175685 11.18 MB/sec execute 247 sec latency 60.391 ms 2024-04-25T04:24:23.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:24.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:24 smithi078 ceph-mon[62942]: pgmap v56: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-04-25T04:24:24.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175952 11.17 MB/sec execute 248 sec latency 56.259 ms 2024-04-25T04:24:24.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:24 smithi112 ceph-mon[50007]: pgmap v56: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.4 MiB/s wr, 431 op/s 2024-04-25T04:24:25.105 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:24 smithi078 systemd[1]: Stopping Ceph osd.0 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:24:25.376 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[32354]: 2024-04-25T04:24:25.293+0000 7ff44901a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:24:25.377 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[32354]: 2024-04-25T04:24:25.293+0000 7ff44901a700 -1 osd.0 75 *** Got signal Terminated *** 2024-04-25T04:24:25.377 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[32354]: 2024-04-25T04:24:25.293+0000 7ff44901a700 -1 osd.0 75 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:24:25.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176186 11.15 MB/sec execute 249 sec latency 386.022 ms 2024-04-25T04:24:25.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:25 smithi078 ceph-mon[62942]: osd.0 marked itself down and dead 2024-04-25T04:24:25.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:25 smithi112 ceph-mon[50007]: osd.0 marked itself down and dead 2024-04-25T04:24:26.355 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:26 smithi078 podman[68664]: 2024-04-25 04:24:26.090773241 +0000 UTC m=+0.940256980 container died 21fbb280387362947b45c328e0d3bc0cf335a8da184db8e7126e1c6aec38b0c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, vcs-type=git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754) 2024-04-25T04:24:26.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:26 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:24:26.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:26 smithi078 ceph-mon[62942]: osdmap e76: 8 total, 7 up, 8 in 2024-04-25T04:24:26.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:26 smithi078 ceph-mon[62942]: pgmap v58: 97 pgs: 13 stale+active+clean, 84 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 453 op/s 2024-04-25T04:24:26.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176358 11.13 MB/sec execute 250 sec latency 1027.416 ms 2024-04-25T04:24:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:26 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:24:26.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:26 smithi112 ceph-mon[50007]: osdmap e76: 8 total, 7 up, 8 in 2024-04-25T04:24:26.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:26 smithi112 ceph-mon[50007]: pgmap v58: 97 pgs: 13 stale+active+clean, 84 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 453 op/s 2024-04-25T04:24:26.952 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:26 smithi078 podman[68664]: 2024-04-25 04:24:26.83453405 +0000 UTC m=+1.684017797 container cleanup 21fbb280387362947b45c328e0d3bc0cf335a8da184db8e7126e1c6aec38b0c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container) 2024-04-25T04:24:26.952 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:26 smithi078 bash[68664]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0 2024-04-25T04:24:27.211 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:27 smithi078 podman[68678]: 2024-04-25 04:24:27.155375482 +0000 UTC m=+1.053207152 container remove 21fbb280387362947b45c328e0d3bc0cf335a8da184db8e7126e1c6aec38b0c0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, name=centos-stream, architecture=x86_64, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, release=754) 2024-04-25T04:24:27.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:27 smithi078 ceph-mon[62942]: osdmap e77: 8 total, 7 up, 8 in 2024-04-25T04:24:27.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176834 11.16 MB/sec execute 251 sec latency 38.817 ms 2024-04-25T04:24:27.857 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:27 smithi078 podman[68763]: 2024-04-25 04:24:27.691542896 +0000 UTC m=+0.030799297 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:24:27.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:27 smithi112 ceph-mon[50007]: osdmap e77: 8 total, 7 up, 8 in 2024-04-25T04:24:27.964 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:27 smithi078 podman[68763]: 2024-04-25 04:24:27.890302726 +0000 UTC m=+0.229559111 container create e2019a5c6518873298d92ebe636d9db7949296f26fee44b15e0d6bf1b430e453 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-25T04:24:28.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:28 smithi078 ceph-mon[62942]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.5 MiB/s wr, 273 op/s 2024-04-25T04:24:28.470 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:28 smithi078 podman[68763]: 2024-04-25 04:24:28.380261526 +0000 UTC m=+0.719517923 container init e2019a5c6518873298d92ebe636d9db7949296f26fee44b15e0d6bf1b430e453 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:24:28.470 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:28 smithi078 podman[68763]: 2024-04-25 04:24:28.387380032 +0000 UTC m=+0.726636419 container start e2019a5c6518873298d92ebe636d9db7949296f26fee44b15e0d6bf1b430e453 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-25T04:24:28.721 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:28 smithi078 podman[68763]: 2024-04-25 04:24:28.571543649 +0000 UTC m=+0.910800036 container attach e2019a5c6518873298d92ebe636d9db7949296f26fee44b15e0d6bf1b430e453 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-25T04:24:28.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177466 11.16 MB/sec execute 252 sec latency 18.973 ms 2024-04-25T04:24:28.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:28 smithi112 ceph-mon[50007]: pgmap v60: 97 pgs: 13 stale+active+clean, 84 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.5 MiB/s wr, 273 op/s 2024-04-25T04:24:28.975 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:28 smithi078 podman[68763]: 2024-04-25 04:24:28.883232611 +0000 UTC m=+1.222489001 container died e2019a5c6518873298d92ebe636d9db7949296f26fee44b15e0d6bf1b430e453 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:24:29.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177946 11.16 MB/sec execute 253 sec latency 35.336 ms 2024-04-25T04:24:30.241 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:30 smithi078 ceph-mon[62942]: pgmap v61: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.1 MiB/s wr, 603 op/s; 219/1770 objects degraded (12.373%) 2024-04-25T04:24:30.241 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:30 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 219/1770 objects degraded (12.373%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:30.492 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:30 smithi078 podman[68788]: 2024-04-25 04:24:30.458034424 +0000 UTC m=+1.568178640 container remove e2019a5c6518873298d92ebe636d9db7949296f26fee44b15e0d6bf1b430e453 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-25T04:24:30.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:30 smithi112 ceph-mon[50007]: pgmap v61: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.1 MiB/s wr, 603 op/s; 219/1770 objects degraded (12.373%) 2024-04-25T04:24:30.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:30 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 219/1770 objects degraded (12.373%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:30.745 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:30 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-25T04:24:30.745 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:30 smithi078 systemd[1]: Stopped Ceph osd.0 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:24:30.745 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:30 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.0.service: Consumed 2min 51.379s CPU time. 2024-04-25T04:24:30.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178555 11.17 MB/sec execute 254 sec latency 34.529 ms 2024-04-25T04:24:31.000 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:30 smithi078 systemd[1]: Starting Ceph osd.0 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:24:31.252 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:31 smithi078 podman[68885]: 2024-04-25 04:24:30.994056558 +0000 UTC m=+0.028303788 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:24:31.252 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:31 smithi078 podman[68885]: 2024-04-25 04:24:31.187668796 +0000 UTC m=+0.221916008 container create 682df3f255bf12b69c75221657c82042c3aecd9cfcd8038299f04b3d36fd201e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:24:31.757 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:31 smithi078 podman[68885]: 2024-04-25 04:24:31.574835592 +0000 UTC m=+0.609082817 container init 682df3f255bf12b69c75221657c82042c3aecd9cfcd8038299f04b3d36fd201e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-25T04:24:31.758 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:31 smithi078 podman[68885]: 2024-04-25 04:24:31.583366672 +0000 UTC m=+0.617613886 container start 682df3f255bf12b69c75221657c82042c3aecd9cfcd8038299f04b3d36fd201e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:24:31.758 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:31 smithi078 podman[68885]: 2024-04-25 04:24:31.741272385 +0000 UTC m=+0.775519611 container attach 682df3f255bf12b69c75221657c82042c3aecd9cfcd8038299f04b3d36fd201e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-25T04:24:31.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179077 11.17 MB/sec execute 255 sec latency 25.448 ms 2024-04-25T04:24:32.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-mon[62942]: pgmap v62: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.3 MiB/s wr, 519 op/s; 219/1770 objects degraded (12.373%) 2024-04-25T04:24:32.342 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:24:32.342 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:24:32.342 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:24:32.342 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:24:32.342 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 bash[68885]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-25T04:24:32.343 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate[68931]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-25T04:24:32.595 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:32 smithi078 podman[68885]: 2024-04-25 04:24:32.34659794 +0000 UTC m=+1.380845152 container died 682df3f255bf12b69c75221657c82042c3aecd9cfcd8038299f04b3d36fd201e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-25T04:24:32.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:32 smithi112 ceph-mon[50007]: pgmap v62: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.3 MiB/s wr, 519 op/s; 219/1770 objects degraded (12.373%) 2024-04-25T04:24:32.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179675 11.20 MB/sec execute 256 sec latency 41.124 ms 2024-04-25T04:24:33.602 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:33 smithi078 podman[69059]: 2024-04-25 04:24:33.42740173 +0000 UTC m=+1.073482456 container remove 682df3f255bf12b69c75221657c82042c3aecd9cfcd8038299f04b3d36fd201e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-25T04:24:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180262 11.23 MB/sec execute 257 sec latency 22.369 ms 2024-04-25T04:24:33.855 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:33 smithi078 podman[69114]: 2024-04-25 04:24:33.62061568 +0000 UTC m=+0.029608435 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:24:33.855 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:33 smithi078 podman[69114]: 2024-04-25 04:24:33.753765051 +0000 UTC m=+0.162757848 container create d2d996c1e21f9419b11c53dcfa10ee59809704f2ed5332742682cba23332d77d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-25T04:24:34.106 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:34 smithi078 podman[69114]: 2024-04-25 04:24:34.097155582 +0000 UTC m=+0.506148335 container init d2d996c1e21f9419b11c53dcfa10ee59809704f2ed5332742682cba23332d77d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-25T04:24:34.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:34 smithi078 ceph-mon[62942]: pgmap v63: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.3 MiB/s wr, 305 op/s; 219/1770 objects degraded (12.373%) 2024-04-25T04:24:34.361 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:34 smithi078 podman[69114]: 2024-04-25 04:24:34.111970422 +0000 UTC m=+0.520963175 container start d2d996c1e21f9419b11c53dcfa10ee59809704f2ed5332742682cba23332d77d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-25T04:24:34.361 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:34 smithi078 ceph-osd[69132]: -- 172.21.15.78:0/2957562078 <== mon.1 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561f73be7680 con 0x561f73bb0c00 2024-04-25T04:24:34.361 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:34 smithi078 bash[69114]: d2d996c1e21f9419b11c53dcfa10ee59809704f2ed5332742682cba23332d77d 2024-04-25T04:24:34.361 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:34 smithi078 systemd[1]: Started Ceph osd.0 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:24:34.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:34 smithi112 ceph-mon[50007]: pgmap v63: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.3 MiB/s wr, 305 op/s; 219/1770 objects degraded (12.373%) 2024-04-25T04:24:34.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180717 11.26 MB/sec execute 258 sec latency 26.679 ms 2024-04-25T04:24:34.863 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:34 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[69128]: 2024-04-25T04:24:34.695+0000 7f968891b7c0 -1 Falling back to public interface 2024-04-25T04:24:35.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:35.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:35.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:35.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:35.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:35.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181138 11.24 MB/sec execute 259 sec latency 38.346 ms 2024-04-25T04:24:36.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:36 smithi078 ceph-mon[62942]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 668 op/s; 226/1809 objects degraded (12.493%) 2024-04-25T04:24:36.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:36 smithi112 ceph-mon[50007]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.1 MiB/s wr, 668 op/s; 226/1809 objects degraded (12.493%) 2024-04-25T04:24:36.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181498 11.23 MB/sec execute 260 sec latency 35.726 ms 2024-04-25T04:24:37.383 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:37 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 226/1809 objects degraded (12.493%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:37.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:37 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 226/1809 objects degraded (12.493%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181917 11.22 MB/sec execute 261 sec latency 40.168 ms 2024-04-25T04:24:38.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:38 smithi078 ceph-mon[62942]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.1 MiB/s wr, 573 op/s; 226/1809 objects degraded (12.493%) 2024-04-25T04:24:38.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:38.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:38.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:38 smithi112 ceph-mon[50007]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.1 MiB/s wr, 573 op/s; 226/1809 objects degraded (12.493%) 2024-04-25T04:24:38.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:38.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:38.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182404 11.23 MB/sec execute 262 sec latency 51.528 ms 2024-04-25T04:24:39.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182876 11.24 MB/sec execute 263 sec latency 47.699 ms 2024-04-25T04:24:40.442 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:40 smithi078 ceph-mon[62942]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 822 op/s; 229/1734 objects degraded (13.206%) 2024-04-25T04:24:40.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:40 smithi112 ceph-mon[50007]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 822 op/s; 229/1734 objects degraded (13.206%) 2024-04-25T04:24:40.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183479 11.26 MB/sec execute 264 sec latency 24.813 ms 2024-04-25T04:24:41.702 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:41 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[69128]: 2024-04-25T04:24:41.564+0000 7f968891b7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-25T04:24:41.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183989 11.29 MB/sec execute 265 sec latency 28.930 ms 2024-04-25T04:24:42.204 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:42 smithi078 ceph-mon[62942]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 602 op/s; 229/1734 objects degraded (13.206%) 2024-04-25T04:24:42.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:42 smithi112 ceph-mon[50007]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 602 op/s; 229/1734 objects degraded (13.206%) 2024-04-25T04:24:42.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184395 11.29 MB/sec execute 266 sec latency 41.080 ms 2024-04-25T04:24:42.972 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:42 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[69128]: 2024-04-25T04:24:42.885+0000 7f968891b7c0 -1 osd.0 75 log_to_monitors true 2024-04-25T04:24:43.230 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:43 smithi078 ceph-mon[62942]: from='osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-25T04:24:43.481 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:24:43 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[69128]: 2024-04-25T04:24:43.231+0000 7f9679d73700 -1 osd.0 75 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:24:43.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:43 smithi112 ceph-mon[50007]: from='osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-25T04:24:43.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184676 11.26 MB/sec execute 267 sec latency 62.525 ms 2024-04-25T04:24:44.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:44 smithi078 ceph-mon[62942]: from='osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-25T04:24:44.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:44 smithi078 ceph-mon[62942]: osdmap e78: 8 total, 7 up, 8 in 2024-04-25T04:24:44.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:44 smithi078 ceph-mon[62942]: from='osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:24:44.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:44 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 229/1734 objects degraded (13.206%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:44.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:44.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:44.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:44 smithi078 ceph-mon[62942]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 816 op/s; 234/1743 objects degraded (13.425%) 2024-04-25T04:24:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:44 smithi112 ceph-mon[50007]: from='osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-25T04:24:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:44 smithi112 ceph-mon[50007]: osdmap e78: 8 total, 7 up, 8 in 2024-04-25T04:24:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:44 smithi112 ceph-mon[50007]: from='osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:24:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:44 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 229/1734 objects degraded (13.206%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:44 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:44 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:44 smithi112 ceph-mon[50007]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 816 op/s; 234/1743 objects degraded (13.425%) 2024-04-25T04:24:44.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184901 11.24 MB/sec execute 268 sec latency 580.784 ms 2024-04-25T04:24:45.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:45 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:24:45.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:45 smithi078 ceph-mon[62942]: osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186] boot 2024-04-25T04:24:45.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:45 smithi078 ceph-mon[62942]: osdmap e79: 8 total, 8 up, 8 in 2024-04-25T04:24:45.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:24:45.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:45 smithi078 ceph-mon[62942]: osdmap e80: 8 total, 8 up, 8 in 2024-04-25T04:24:45.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:45 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:24:45.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:45 smithi112 ceph-mon[50007]: osd.0 [v2:172.21.15.78:6802/602428186,v1:172.21.15.78:6803/602428186] boot 2024-04-25T04:24:45.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:45 smithi112 ceph-mon[50007]: osdmap e79: 8 total, 8 up, 8 in 2024-04-25T04:24:45.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-25T04:24:45.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:45 smithi112 ceph-mon[50007]: osdmap e80: 8 total, 8 up, 8 in 2024-04-25T04:24:45.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184964 11.20 MB/sec execute 269 sec latency 1580.818 ms 2024-04-25T04:24:46.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:46.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:46.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:46 smithi078 ceph-mon[62942]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.8 MiB/s wr, 527 op/s; 221/1725 objects degraded (12.812%) 2024-04-25T04:24:46.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:46 smithi078 ceph-mon[62942]: osdmap e81: 8 total, 8 up, 8 in 2024-04-25T04:24:46.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:46.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:46.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:46 smithi112 ceph-mon[50007]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.8 MiB/s wr, 527 op/s; 221/1725 objects degraded (12.812%) 2024-04-25T04:24:46.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:46 smithi112 ceph-mon[50007]: osdmap e81: 8 total, 8 up, 8 in 2024-04-25T04:24:46.836 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184974 11.16 MB/sec execute 270 sec latency 2456.385 ms 2024-04-25T04:24:47.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185157 11.13 MB/sec execute 271 sec latency 478.387 ms 2024-04-25T04:24:48.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:48 smithi078 ceph-mon[62942]: osdmap e82: 8 total, 8 up, 8 in 2024-04-25T04:24:48.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:48 smithi078 ceph-mon[62942]: pgmap v75: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 557 op/s; 221/1725 objects degraded (12.812%) 2024-04-25T04:24:48.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:48 smithi112 ceph-mon[50007]: osdmap e82: 8 total, 8 up, 8 in 2024-04-25T04:24:48.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:48 smithi112 ceph-mon[50007]: pgmap v75: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 557 op/s; 221/1725 objects degraded (12.812%) 2024-04-25T04:24:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185752 11.15 MB/sec execute 272 sec latency 1032.075 ms 2024-04-25T04:24:49.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:49 smithi078 ceph-mon[62942]: osdmap e83: 8 total, 8 up, 8 in 2024-04-25T04:24:49.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:49 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 221/1725 objects degraded (12.812%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:49 smithi112 ceph-mon[50007]: osdmap e83: 8 total, 8 up, 8 in 2024-04-25T04:24:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:49 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 221/1725 objects degraded (12.812%), 29 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:49.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186439 11.17 MB/sec execute 273 sec latency 875.864 ms 2024-04-25T04:24:50.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: osdmap e84: 8 total, 8 up, 8 in 2024-04-25T04:24:50.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:24:50.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: pgmap v78: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 474 op/s; 175/1596 objects degraded (10.965%); 3.3 MiB/s, 37 objects/s recovering 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:50.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: osdmap e84: 8 total, 8 up, 8 in 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: pgmap v78: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 474 op/s; 175/1596 objects degraded (10.965%); 3.3 MiB/s, 37 objects/s recovering 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:50.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:50.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:24:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186834 11.17 MB/sec execute 274 sec latency 124.966 ms 2024-04-25T04:24:51.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:51 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:24:51.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:51 smithi078 ceph-mon[62942]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-25T04:24:51.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:51 smithi078 ceph-mon[62942]: osdmap e85: 8 total, 8 up, 8 in 2024-04-25T04:24:51.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:51 smithi078 ceph-mon[62942]: osdmap e86: 8 total, 8 up, 8 in 2024-04-25T04:24:51.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:51 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:24:51.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:51 smithi112 ceph-mon[50007]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-25T04:24:51.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:51 smithi112 ceph-mon[50007]: osdmap e85: 8 total, 8 up, 8 in 2024-04-25T04:24:51.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:51 smithi112 ceph-mon[50007]: osdmap e86: 8 total, 8 up, 8 in 2024-04-25T04:24:51.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187161 11.17 MB/sec execute 275 sec latency 681.158 ms 2024-04-25T04:24:52.346 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:24:52.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:52 smithi078 ceph-mon[62942]: pgmap v81: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 474 op/s; 175/1596 objects degraded (10.965%); 3.3 MiB/s, 37 objects/s recovering 2024-04-25T04:24:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:52 smithi112 ceph-mon[50007]: pgmap v81: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 474 op/s; 175/1596 objects degraded (10.965%); 3.3 MiB/s, 37 objects/s recovering 2024-04-25T04:24:52.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187494 11.17 MB/sec execute 276 sec latency 395.447 ms 2024-04-25T04:24:53.482 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:24:53.482 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 9s ago 13m 25.7M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:24:53.482 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (60s) 9s ago 13m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:24:53.482 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (54s) 49s ago 12m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:24:53.482 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 9s ago 12m 54.3M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (7m) 9s ago 7m 40.4M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (7m) 49s ago 7m 14.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (3m) 9s ago 14m 501M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (2m) 49s ago 12m 450M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 9s ago 14m 45.4M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (117s) 49s ago 12m 35.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (6m) 9s ago 6m 278M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (6m) 49s ago 6m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 9s ago 12m 17.6M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (3m) 49s ago 12m 16.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (19s) 9s ago 11m 15.0M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 9s ago 11m 556M 1936M 17.2.0 e1d6a67b021e 5ac6ddda81b2 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 9s ago 10m 423M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:24:53.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 9s ago 10m 548M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:24:53.484 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (9m) 49s ago 9m 528M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:24:53.484 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (9m) 49s ago 9m 381M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:24:53.484 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (9m) 49s ago 9m 556M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:24:53.484 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (8m) 49s ago 8m 466M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:24:53.484 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (2m) 9s ago 12m 71.0M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:24:53.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:53 smithi078 ceph-mon[62942]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:53.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:53 smithi078 ceph-mon[62942]: osdmap e87: 8 total, 8 up, 8 in 2024-04-25T04:24:53.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:53 smithi078 ceph-mon[62942]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:53.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:53.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:53 smithi112 ceph-mon[50007]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:53 smithi112 ceph-mon[50007]: osdmap e87: 8 total, 8 up, 8 in 2024-04-25T04:24:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:53 smithi112 ceph-mon[50007]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:24:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:24:53.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187817 11.17 MB/sec execute 277 sec latency 976.577 ms 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:54.133 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 5 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:24:54.134 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:24:54.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:54 smithi078 ceph-mon[62942]: osdmap e88: 8 total, 8 up, 8 in 2024-04-25T04:24:54.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:54 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 175/1596 objects degraded (10.965%), 7 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:54.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:54 smithi078 ceph-mon[62942]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:54.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:54 smithi078 ceph-mon[62942]: pgmap v84: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 91 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s wr, 179 op/s; 80/1650 objects degraded (4.848%); 0 B/s, 23 objects/s recovering 2024-04-25T04:24:54.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:54 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1779367799' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:24:54.711 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:24:54.712 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:24:54.712 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "7/23 daemons upgraded", 2024-04-25T04:24:54.712 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:24:54.712 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:24:54.712 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:24:54.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188455 11.17 MB/sec execute 278 sec latency 33.173 ms 2024-04-25T04:24:54.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:54 smithi112 ceph-mon[50007]: osdmap e88: 8 total, 8 up, 8 in 2024-04-25T04:24:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:54 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 175/1596 objects degraded (10.965%), 7 pgs degraded (PG_DEGRADED) 2024-04-25T04:24:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:54 smithi112 ceph-mon[50007]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:54 smithi112 ceph-mon[50007]: pgmap v84: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 91 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s wr, 179 op/s; 80/1650 objects degraded (4.848%); 0 B/s, 23 objects/s recovering 2024-04-25T04:24:54.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:54 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1779367799' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:24:55.399 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 80/1650 objects degraded (4.848%), 3 pgs degraded 2024-04-25T04:24:55.399 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 80/1650 objects degraded (4.848%), 3 pgs degraded 2024-04-25T04:24:55.399 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-04-25T04:24:55.400 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.10 is active+recovery_wait+undersized+degraded+remapped, acting [6,5] 2024-04-25T04:24:55.400 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-04-25T04:24:55.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:55 smithi078 ceph-mon[62942]: from='client.44181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:55.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:55 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2863647256' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:24:55.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189018 11.18 MB/sec execute 279 sec latency 25.170 ms 2024-04-25T04:24:55.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:55 smithi112 ceph-mon[50007]: from='client.44181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:24:55.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:55 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2863647256' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:24:56.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:56 smithi078 ceph-mon[62942]: pgmap v85: 97 pgs: 2 peering, 95 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 503 op/s; 0 B/s, 30 objects/s recovering 2024-04-25T04:24:56.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189690 11.22 MB/sec execute 280 sec latency 59.824 ms 2024-04-25T04:24:56.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:56 smithi112 ceph-mon[50007]: pgmap v85: 97 pgs: 2 peering, 95 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 503 op/s; 0 B/s, 30 objects/s recovering 2024-04-25T04:24:57.435 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:57 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/1650 objects degraded (4.848%), 3 pgs degraded) 2024-04-25T04:24:57.436 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:57 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:24:57.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190245 11.22 MB/sec execute 281 sec latency 33.762 ms 2024-04-25T04:24:57.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:57 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/1650 objects degraded (4.848%), 3 pgs degraded) 2024-04-25T04:24:57.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:57 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:24:58.442 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:24:58 smithi078 ceph-mon[62942]: pgmap v86: 97 pgs: 2 peering, 95 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 0 B/s, 25 objects/s recovering 2024-04-25T04:24:58.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190625 11.23 MB/sec execute 282 sec latency 47.799 ms 2024-04-25T04:24:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:24:58 smithi112 ceph-mon[50007]: pgmap v86: 97 pgs: 2 peering, 95 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 0 B/s, 25 objects/s recovering 2024-04-25T04:24:59.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191158 11.25 MB/sec execute 283 sec latency 28.702 ms 2024-04-25T04:25:00.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:00 smithi078 ceph-mon[62942]: pgmap v87: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 861 op/s; 0 B/s, 21 objects/s recovering 2024-04-25T04:25:00.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:00 smithi112 ceph-mon[50007]: pgmap v87: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 861 op/s; 0 B/s, 21 objects/s recovering 2024-04-25T04:25:00.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191377 11.24 MB/sec execute 284 sec latency 53.455 ms 2024-04-25T04:25:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191582 11.22 MB/sec execute 285 sec latency 66.739 ms 2024-04-25T04:25:02.435 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:02 smithi078 ceph-mon[62942]: pgmap v88: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.3 MiB/s wr, 708 op/s; 0 B/s, 17 objects/s recovering 2024-04-25T04:25:02.653 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:02 smithi112 ceph-mon[50007]: pgmap v88: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.3 MiB/s wr, 708 op/s; 0 B/s, 17 objects/s recovering 2024-04-25T04:25:02.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191827 11.19 MB/sec execute 286 sec latency 75.077 ms 2024-04-25T04:25:03.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192145 11.16 MB/sec execute 287 sec latency 86.061 ms 2024-04-25T04:25:04.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:04 smithi078 ceph-mon[62942]: pgmap v89: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 655 op/s; 0 B/s, 7 objects/s recovering 2024-04-25T04:25:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:04 smithi112 ceph-mon[50007]: pgmap v89: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 655 op/s; 0 B/s, 7 objects/s recovering 2024-04-25T04:25:04.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192402 11.14 MB/sec execute 288 sec latency 58.602 ms 2024-04-25T04:25:05.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:05 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:25:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:05 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:25:05.837 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192677 11.14 MB/sec execute 289 sec latency 59.336 ms 2024-04-25T04:25:06.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:06 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:25:06.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:06 smithi078 ceph-mon[62942]: Upgrade: osd.1 is safe to restart 2024-04-25T04:25:06.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:06 smithi078 ceph-mon[62942]: pgmap v90: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 694 op/s; 0 B/s, 6 objects/s recovering 2024-04-25T04:25:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:06 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-25T04:25:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:06 smithi112 ceph-mon[50007]: Upgrade: osd.1 is safe to restart 2024-04-25T04:25:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:06 smithi112 ceph-mon[50007]: pgmap v90: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 694 op/s; 0 B/s, 6 objects/s recovering 2024-04-25T04:25:06.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192996 11.14 MB/sec execute 290 sec latency 55.382 ms 2024-04-25T04:25:07.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193336 11.13 MB/sec execute 291 sec latency 91.641 ms 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: Upgrade: Updating osd.1 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: Deploying daemon osd.1 on smithi078 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: pgmap v91: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:08.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:08.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:08.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193781 11.12 MB/sec execute 292 sec latency 30.605 ms 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: Upgrade: Updating osd.1 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: Deploying daemon osd.1 on smithi078 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: pgmap v91: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:09.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194178 11.13 MB/sec execute 293 sec latency 32.679 ms 2024-04-25T04:25:10.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:10 smithi078 ceph-mon[62942]: pgmap v92: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 719 op/s 2024-04-25T04:25:10.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:10 smithi112 ceph-mon[50007]: pgmap v92: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 719 op/s 2024-04-25T04:25:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194551 11.13 MB/sec execute 294 sec latency 39.264 ms 2024-04-25T04:25:11.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194869 11.13 MB/sec execute 295 sec latency 47.400 ms 2024-04-25T04:25:12.438 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:12 smithi078 ceph-mon[62942]: pgmap v93: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 386 op/s 2024-04-25T04:25:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:12 smithi112 ceph-mon[50007]: pgmap v93: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 386 op/s 2024-04-25T04:25:12.695 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:12 smithi078 systemd[1]: Stopping Ceph osd.1 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:25:12.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195159 11.13 MB/sec execute 296 sec latency 65.945 ms 2024-04-25T04:25:12.946 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:12 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[36314]: 2024-04-25T04:25:12.885+0000 7f1eb153b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:25:12.947 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:12 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[36314]: 2024-04-25T04:25:12.885+0000 7f1eb153b700 -1 osd.1 88 *** Got signal Terminated *** 2024-04-25T04:25:12.947 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:12 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[36314]: 2024-04-25T04:25:12.885+0000 7f1eb153b700 -1 osd.1 88 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:25:13.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:13 smithi078 ceph-mon[62942]: osd.1 marked itself down and dead 2024-04-25T04:25:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:13 smithi112 ceph-mon[50007]: osd.1 marked itself down and dead 2024-04-25T04:25:13.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195415 11.10 MB/sec execute 297 sec latency 581.564 ms 2024-04-25T04:25:13.984 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:13 smithi078 podman[71237]: 2024-04-25 04:25:13.901751635 +0000 UTC m=+1.230651273 container died 5ac6ddda81b2210b2d502a62d190bb945cbaf4969ec5791b4a8369c76bee9b4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., name=centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, architecture=x86_64, RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public, vcs-type=git, GIT_BRANCH=HEAD) 2024-04-25T04:25:14.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:14 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:25:14.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:14 smithi078 ceph-mon[62942]: osdmap e89: 8 total, 7 up, 8 in 2024-04-25T04:25:14.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:14 smithi078 ceph-mon[62942]: pgmap v95: 97 pgs: 4 peering, 12 stale+active+clean, 81 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 585 op/s 2024-04-25T04:25:14.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:14 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:25:14.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:14 smithi112 ceph-mon[50007]: osdmap e89: 8 total, 7 up, 8 in 2024-04-25T04:25:14.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:14 smithi112 ceph-mon[50007]: pgmap v95: 97 pgs: 4 peering, 12 stale+active+clean, 81 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.1 MiB/s wr, 585 op/s 2024-04-25T04:25:14.757 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:14 smithi078 podman[71237]: 2024-04-25 04:25:14.53967061 +0000 UTC m=+1.868570240 container cleanup 5ac6ddda81b2210b2d502a62d190bb945cbaf4969ec5791b4a8369c76bee9b4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_BRANCH=HEAD, name=centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=) 2024-04-25T04:25:14.757 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:14 smithi078 bash[71237]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1 2024-04-25T04:25:14.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195783 11.08 MB/sec execute 298 sec latency 992.017 ms 2024-04-25T04:25:15.017 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:14 smithi078 podman[71254]: 2024-04-25 04:25:14.906223101 +0000 UTC m=+0.998049216 container remove 5ac6ddda81b2210b2d502a62d190bb945cbaf4969ec5791b4a8369c76bee9b4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-25T04:25:15.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:15 smithi078 ceph-mon[62942]: osdmap e90: 8 total, 7 up, 8 in 2024-04-25T04:25:15.528 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:15 smithi078 podman[71340]: 2024-04-25 04:25:15.338846108 +0000 UTC m=+0.031215500 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:25:15.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:15 smithi112 ceph-mon[50007]: osdmap e90: 8 total, 7 up, 8 in 2024-04-25T04:25:15.790 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:15 smithi078 podman[71340]: 2024-04-25 04:25:15.670165743 +0000 UTC m=+0.362535128 container create a70d2a3483e4a771f6a831fd9baa13ffc9f155f5a6f4646071e9f564f765e1a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-25T04:25:15.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196123 11.07 MB/sec execute 299 sec latency 44.622 ms 2024-04-25T04:25:16.045 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:16 smithi078 podman[71340]: 2024-04-25 04:25:16.024596751 +0000 UTC m=+0.716966147 container init a70d2a3483e4a771f6a831fd9baa13ffc9f155f5a6f4646071e9f564f765e1a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:25:16.045 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:16 smithi078 podman[71340]: 2024-04-25 04:25:16.029770426 +0000 UTC m=+0.722139813 container start a70d2a3483e4a771f6a831fd9baa13ffc9f155f5a6f4646071e9f564f765e1a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-04-25T04:25:16.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:16 smithi078 ceph-mon[62942]: pgmap v97: 97 pgs: 35 peering, 1 stale+active+clean, 61 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-04-25T04:25:16.336 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:16 smithi078 podman[71340]: 2024-04-25 04:25:16.184495332 +0000 UTC m=+0.876864724 container attach a70d2a3483e4a771f6a831fd9baa13ffc9f155f5a6f4646071e9f564f765e1a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-25T04:25:16.589 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:16 smithi078 podman[71340]: 2024-04-25 04:25:16.474523508 +0000 UTC m=+1.166892905 container died a70d2a3483e4a771f6a831fd9baa13ffc9f155f5a6f4646071e9f564f765e1a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-25T04:25:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:16 smithi112 ceph-mon[50007]: pgmap v97: 97 pgs: 35 peering, 1 stale+active+clean, 61 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.1 MiB/s wr, 612 op/s 2024-04-25T04:25:16.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196565 11.09 MB/sec execute 300 sec latency 65.023 ms 2024-04-25T04:25:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:17 smithi078 ceph-mon[62942]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-25T04:25:17.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:17 smithi112 ceph-mon[50007]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-25T04:25:17.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197283 11.11 MB/sec execute 301 sec latency 26.390 ms 2024-04-25T04:25:18.104 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:18 smithi078 podman[71364]: 2024-04-25 04:25:18.01196453 +0000 UTC m=+1.531736652 container remove a70d2a3483e4a771f6a831fd9baa13ffc9f155f5a6f4646071e9f564f765e1a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:25:18.104 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:18 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-25T04:25:18.104 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:18 smithi078 systemd[1]: Stopped Ceph osd.1 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:25:18.104 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:18 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.1.service: Consumed 3min 22.268s CPU time. 2024-04-25T04:25:18.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:18 smithi078 ceph-mon[62942]: pgmap v98: 97 pgs: 35 peering, 1 stale+active+clean, 61 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 303 op/s 2024-04-25T04:25:18.606 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:18 smithi078 systemd[1]: Starting Ceph osd.1 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:25:18.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:18 smithi112 ceph-mon[50007]: pgmap v98: 97 pgs: 35 peering, 1 stale+active+clean, 61 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 303 op/s 2024-04-25T04:25:18.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197795 11.13 MB/sec execute 302 sec latency 36.837 ms 2024-04-25T04:25:18.857 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:18 smithi078 podman[71462]: 2024-04-25 04:25:18.538853429 +0000 UTC m=+0.021580921 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:25:18.857 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:18 smithi078 podman[71462]: 2024-04-25 04:25:18.680315497 +0000 UTC m=+0.163042971 container create 01981026f2203c95a99034be13ec3a93120e7eccf05b04bd3d54590fba16eb30 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-25T04:25:19.362 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 podman[71462]: 2024-04-25 04:25:19.109968942 +0000 UTC m=+0.592696426 container init 01981026f2203c95a99034be13ec3a93120e7eccf05b04bd3d54590fba16eb30 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:25:19.362 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 podman[71462]: 2024-04-25 04:25:19.118777675 +0000 UTC m=+0.601505157 container start 01981026f2203c95a99034be13ec3a93120e7eccf05b04bd3d54590fba16eb30 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-25T04:25:19.362 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 podman[71462]: 2024-04-25 04:25:19.320379681 +0000 UTC m=+0.803107160 container attach 01981026f2203c95a99034be13ec3a93120e7eccf05b04bd3d54590fba16eb30 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-25T04:25:19.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198225 11.13 MB/sec execute 303 sec latency 45.696 ms 2024-04-25T04:25:19.865 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:25:19.865 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 bash[71462]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-25T04:25:19.866 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-25T04:25:19.867 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-25T04:25:19.867 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-25T04:25:19.867 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate[71477]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-25T04:25:20.117 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:19 smithi078 podman[71462]: 2024-04-25 04:25:19.880763798 +0000 UTC m=+1.363491278 container died 01981026f2203c95a99034be13ec3a93120e7eccf05b04bd3d54590fba16eb30 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-25T04:25:20.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:20 smithi078 ceph-mon[62942]: pgmap v99: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.7 MiB/s wr, 722 op/s; 189/1788 objects degraded (10.570%) 2024-04-25T04:25:20.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:20 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 189/1788 objects degraded (10.570%), 24 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:20.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:20 smithi078 ceph-mon[62942]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-04-25T04:25:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:20 smithi112 ceph-mon[50007]: pgmap v99: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.7 MiB/s wr, 722 op/s; 189/1788 objects degraded (10.570%) 2024-04-25T04:25:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:20 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 189/1788 objects degraded (10.570%), 24 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:20.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:20 smithi112 ceph-mon[50007]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-04-25T04:25:20.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198607 11.14 MB/sec execute 304 sec latency 28.658 ms 2024-04-25T04:25:21.126 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:20 smithi078 podman[71604]: 2024-04-25 04:25:20.895272482 +0000 UTC m=+1.008363082 container remove 01981026f2203c95a99034be13ec3a93120e7eccf05b04bd3d54590fba16eb30 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-25T04:25:21.378 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:21 smithi078 podman[71658]: 2024-04-25 04:25:21.057802623 +0000 UTC m=+0.028259602 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:25:21.378 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:21 smithi078 podman[71658]: 2024-04-25 04:25:21.213572229 +0000 UTC m=+0.184029191 container create 5d98f4fa8718fa5c046f802f5292890b4f990bbf3fcaf0dc62c8bf800446e8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-25T04:25:21.631 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:21 smithi078 podman[71658]: 2024-04-25 04:25:21.509302586 +0000 UTC m=+0.479759552 container init 5d98f4fa8718fa5c046f802f5292890b4f990bbf3fcaf0dc62c8bf800446e8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:25:21.631 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:21 smithi078 podman[71658]: 2024-04-25 04:25:21.514719901 +0000 UTC m=+0.485176866 container start 5d98f4fa8718fa5c046f802f5292890b4f990bbf3fcaf0dc62c8bf800446e8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:25:21.631 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:21 smithi078 ceph-osd[71676]: -- 172.21.15.78:0/530517646 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5580f40a3680 con 0x5580f406d000 2024-04-25T04:25:21.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198914 11.12 MB/sec execute 305 sec latency 45.723 ms 2024-04-25T04:25:21.892 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:21 smithi078 bash[71658]: 5d98f4fa8718fa5c046f802f5292890b4f990bbf3fcaf0dc62c8bf800446e8ac 2024-04-25T04:25:21.892 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:21 smithi078 systemd[1]: Started Ceph osd.1 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:25:22.155 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:22 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:25:22.100+0000 7f53f65257c0 -1 Falling back to public interface 2024-04-25T04:25:22.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:22.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:22.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:25:22.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:22 smithi078 ceph-mon[62942]: pgmap v100: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 649 op/s; 189/1788 objects degraded (10.570%) 2024-04-25T04:25:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199228 11.09 MB/sec execute 306 sec latency 57.671 ms 2024-04-25T04:25:23.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:23.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:23.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:25:23.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:22 smithi112 ceph-mon[50007]: pgmap v100: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.9 MiB/s wr, 649 op/s; 189/1788 objects degraded (10.570%) 2024-04-25T04:25:23.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199477 11.07 MB/sec execute 307 sec latency 64.337 ms 2024-04-25T04:25:24.211 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:24.211 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:24.211 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:24 smithi078 ceph-mon[62942]: pgmap v101: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 6.6 MiB/s wr, 654 op/s; 185/1713 objects degraded (10.800%) 2024-04-25T04:25:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:24 smithi112 ceph-mon[50007]: pgmap v101: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 6.6 MiB/s wr, 654 op/s; 185/1713 objects degraded (10.800%) 2024-04-25T04:25:24.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199771 11.07 MB/sec execute 308 sec latency 57.677 ms 2024-04-25T04:25:25.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200184 11.08 MB/sec execute 309 sec latency 52.331 ms 2024-04-25T04:25:25.911 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:25:26.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:26 smithi078 ceph-mon[62942]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:26.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:26 smithi078 ceph-mon[62942]: pgmap v102: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 178/1677 objects degraded (10.614%) 2024-04-25T04:25:26.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:26 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 178/1677 objects degraded (10.614%), 24 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:26 smithi112 ceph-mon[50007]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:26 smithi112 ceph-mon[50007]: pgmap v102: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 178/1677 objects degraded (10.614%) 2024-04-25T04:25:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:26 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 178/1677 objects degraded (10.614%), 24 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:26.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200611 11.07 MB/sec execute 310 sec latency 39.610 ms 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 43s ago 13m 25.7M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (94s) 43s ago 13m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (87s) 82s ago 13m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 43s ago 13m 54.3M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (7m) 43s ago 7m 40.4M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (7m) 82s ago 7m 14.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (4m) 43s ago 15m 501M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (3m) 82s ago 13m 450M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 43s ago 15m 45.4M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (2m) 82s ago 12m 35.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:25:26.973 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (7m) 43s ago 7m 278M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (7m) 82s ago 7m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 43s ago 13m 17.6M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (4m) 82s ago 12m 16.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (53s) 43s ago 12m 15.0M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 starting - - - 1936M 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 43s ago 11m 423M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 43s ago 10m 548M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (10m) 82s ago 10m 528M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (10m) 82s ago 9m 381M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (9m) 82s ago 9m 556M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (9m) 82s ago 9m 466M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:25:26.974 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 43s ago 12m 71.0M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:25:27.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:27 smithi078 ceph-mon[62942]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:27.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:27 smithi078 ceph-mon[62942]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:27.596 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:25:27.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:25:27.598 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:25:27.598 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-25T04:25:27.598 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 5 2024-04-25T04:25:27.598 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:25:27.598 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:25:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:27 smithi112 ceph-mon[50007]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:27.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:27 smithi112 ceph-mon[50007]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:27.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200952 11.07 MB/sec execute 311 sec latency 45.396 ms 2024-04-25T04:25:28.083 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:25:28.083 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:25:28.083 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:25:28.083 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:25:28.083 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:25:28.083 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:25:28.084 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:25:28.084 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:25:28.084 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:25:28.084 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "7/23 daemons upgraded", 2024-04-25T04:25:28.084 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:25:28.084 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:25:28.084 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:25:28.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:28 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/4121156949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:28.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:28 smithi078 ceph-mon[62942]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:28.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:28 smithi078 ceph-mon[62942]: pgmap v103: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 498 op/s; 178/1677 objects degraded (10.614%) 2024-04-25T04:25:28.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:28 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/4121156949' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:28.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:28 smithi112 ceph-mon[50007]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:28.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:28 smithi112 ceph-mon[50007]: pgmap v103: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.9 MiB/s wr, 498 op/s; 178/1677 objects degraded (10.614%) 2024-04-25T04:25:28.662 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 178/1677 objects degraded (10.614%), 24 pgs degraded 2024-04-25T04:25:28.662 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: osd.1 (root=default,host=smithi078) is down 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 178/1677 objects degraded (10.614%), 24 pgs degraded 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-04-25T04:25:28.663 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-04-25T04:25:28.664 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-04-25T04:25:28.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201255 11.07 MB/sec execute 312 sec latency 35.228 ms 2024-04-25T04:25:29.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:29 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1923232015' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:25:29.587 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:29 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:25:29.353+0000 7f53f65257c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-25T04:25:29.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:29 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1923232015' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:25:29.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201590 11.07 MB/sec execute 313 sec latency 46.791 ms 2024-04-25T04:25:30.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:30 smithi078 ceph-mon[62942]: pgmap v104: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 709 op/s; 170/1683 objects degraded (10.101%) 2024-04-25T04:25:30.487 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:30 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:25:30.479+0000 7f53f65257c0 -1 osd.1 88 log_to_monitors true 2024-04-25T04:25:30.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:30 smithi112 ceph-mon[50007]: pgmap v104: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 709 op/s; 170/1683 objects degraded (10.101%) 2024-04-25T04:25:30.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201856 11.07 MB/sec execute 314 sec latency 53.035 ms 2024-04-25T04:25:31.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:31 smithi078 ceph-mon[62942]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T04:25:31.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:31 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:31.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:31 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:31.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:31 smithi112 ceph-mon[50007]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-25T04:25:31.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:31 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:31.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:31 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:31.844 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202048 11.04 MB/sec execute 315 sec latency 108.947 ms 2024-04-25T04:25:31.943 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:25:31 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:25:31.767+0000 7f53e3975700 -1 osd.1 88 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:25:32.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:32 smithi078 ceph-mon[62942]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-25T04:25:32.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:32 smithi078 ceph-mon[62942]: osdmap e91: 8 total, 7 up, 8 in 2024-04-25T04:25:32.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:32 smithi078 ceph-mon[62942]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:25:32.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:32 smithi078 ceph-mon[62942]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' 2024-04-25T04:25:32.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:32 smithi078 ceph-mon[62942]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 515 op/s; 170/1683 objects degraded (10.101%) 2024-04-25T04:25:32.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:32 smithi112 ceph-mon[50007]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-25T04:25:32.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:32 smithi112 ceph-mon[50007]: osdmap e91: 8 total, 7 up, 8 in 2024-04-25T04:25:32.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:32 smithi112 ceph-mon[50007]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:25:32.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:32 smithi112 ceph-mon[50007]: from='osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993]' entity='osd.1' 2024-04-25T04:25:32.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:32 smithi112 ceph-mon[50007]: pgmap v106: 97 pgs: 24 active+undersized+degraded, 15 active+undersized, 58 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 515 op/s; 170/1683 objects degraded (10.101%) 2024-04-25T04:25:32.844 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202407 11.03 MB/sec execute 316 sec latency 89.530 ms 2024-04-25T04:25:33.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:33.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:33.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:33 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:25:33.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:33 smithi078 ceph-mon[62942]: osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993] boot 2024-04-25T04:25:33.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:33 smithi078 ceph-mon[62942]: osdmap e92: 8 total, 8 up, 8 in 2024-04-25T04:25:33.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:25:33.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:33.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:33.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:33 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:25:33.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:33 smithi112 ceph-mon[50007]: osd.1 [v2:172.21.15.78:6810/4282062993,v1:172.21.15.78:6811/4282062993] boot 2024-04-25T04:25:33.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:33 smithi112 ceph-mon[50007]: osdmap e92: 8 total, 8 up, 8 in 2024-04-25T04:25:33.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-25T04:25:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202407 10.99 MB/sec execute 317 sec latency 1089.558 ms 2024-04-25T04:25:34.509 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:34 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 170/1683 objects degraded (10.101%), 24 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:34.509 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:34 smithi078 ceph-mon[62942]: osdmap e93: 8 total, 8 up, 8 in 2024-04-25T04:25:34.509 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:34 smithi078 ceph-mon[62942]: pgmap v109: 97 pgs: 6 remapped+peering, 3 peering, 15 active+undersized+degraded, 7 active+undersized, 66 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 482 op/s; 88/1635 objects degraded (5.382%) 2024-04-25T04:25:34.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:34 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 170/1683 objects degraded (10.101%), 24 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:34.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:34 smithi112 ceph-mon[50007]: osdmap e93: 8 total, 8 up, 8 in 2024-04-25T04:25:34.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:34 smithi112 ceph-mon[50007]: pgmap v109: 97 pgs: 6 remapped+peering, 3 peering, 15 active+undersized+degraded, 7 active+undersized, 66 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 482 op/s; 88/1635 objects degraded (5.382%) 2024-04-25T04:25:34.844 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202675 10.97 MB/sec execute 318 sec latency 1755.665 ms 2024-04-25T04:25:35.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:35 smithi078 ceph-mon[62942]: osdmap e94: 8 total, 8 up, 8 in 2024-04-25T04:25:35.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203012 10.95 MB/sec execute 319 sec latency 382.005 ms 2024-04-25T04:25:35.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:35 smithi112 ceph-mon[50007]: osdmap e94: 8 total, 8 up, 8 in 2024-04-25T04:25:36.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:36 smithi078 ceph-mon[62942]: osdmap e95: 8 total, 8 up, 8 in 2024-04-25T04:25:36.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:36 smithi078 ceph-mon[62942]: pgmap v112: 97 pgs: 72 active+clean, 4 active+undersized, 6 active+undersized+degraded, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 9 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.5 MiB/s wr, 421 op/s; 60/1620 objects degraded (3.704%); 0 B/s, 3 objects/s recovering 2024-04-25T04:25:36.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203258 10.95 MB/sec execute 320 sec latency 1024.905 ms 2024-04-25T04:25:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:36 smithi112 ceph-mon[50007]: osdmap e95: 8 total, 8 up, 8 in 2024-04-25T04:25:36.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:36 smithi112 ceph-mon[50007]: pgmap v112: 97 pgs: 72 active+clean, 4 active+undersized, 6 active+undersized+degraded, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 9 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.5 MiB/s wr, 421 op/s; 60/1620 objects degraded (3.704%); 0 B/s, 3 objects/s recovering 2024-04-25T04:25:37.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:37 smithi078 ceph-mon[62942]: osdmap e96: 8 total, 8 up, 8 in 2024-04-25T04:25:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203860 10.95 MB/sec execute 321 sec latency 861.540 ms 2024-04-25T04:25:37.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:37 smithi112 ceph-mon[50007]: osdmap e96: 8 total, 8 up, 8 in 2024-04-25T04:25:38.499 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: osdmap e97: 8 total, 8 up, 8 in 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: pgmap v115: 97 pgs: 82 active+clean, 4 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 8 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 334 op/s; 30/1626 objects degraded (1.845%); 22 B/s, 9 objects/s recovering 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:25:38.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:38.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:38.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:38.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:38.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 60/1620 objects degraded (3.704%), 9 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:38.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:38 smithi078 ceph-mon[62942]: osdmap e98: 8 total, 8 up, 8 in 2024-04-25T04:25:38.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204193 10.95 MB/sec execute 322 sec latency 928.821 ms 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: osdmap e97: 8 total, 8 up, 8 in 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: pgmap v115: 97 pgs: 82 active+clean, 4 peering, 2 active+recovery_wait+degraded, 1 active+recovering+degraded, 8 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.2 MiB/s wr, 334 op/s; 30/1626 objects degraded (1.845%); 22 B/s, 9 objects/s recovering 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:38.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:25:38.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:38.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:38.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:25:38.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:38.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 60/1620 objects degraded (3.704%), 9 pgs degraded (PG_DEGRADED) 2024-04-25T04:25:38.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:38 smithi112 ceph-mon[50007]: osdmap e98: 8 total, 8 up, 8 in 2024-04-25T04:25:39.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:39 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:39.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:39 smithi078 ceph-mon[62942]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-25T04:25:39.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:39 smithi078 ceph-mon[62942]: osdmap e99: 8 total, 8 up, 8 in 2024-04-25T04:25:39.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204721 10.98 MB/sec execute 323 sec latency 489.554 ms 2024-04-25T04:25:39.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:39 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:39.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:39 smithi112 ceph-mon[50007]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-25T04:25:39.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:39 smithi112 ceph-mon[50007]: osdmap e99: 8 total, 8 up, 8 in 2024-04-25T04:25:40.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:40 smithi078 ceph-mon[62942]: pgmap v118: 97 pgs: 1 active+undersized+remapped, 92 active+clean, 3 peering, 1 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 670 op/s; 1.7 MiB/s, 51 objects/s recovering 2024-04-25T04:25:40.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:40 smithi078 ceph-mon[62942]: osdmap e100: 8 total, 8 up, 8 in 2024-04-25T04:25:40.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205148 10.99 MB/sec execute 324 sec latency 785.477 ms 2024-04-25T04:25:40.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:40 smithi112 ceph-mon[50007]: pgmap v118: 97 pgs: 1 active+undersized+remapped, 92 active+clean, 3 peering, 1 remapped+peering; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 670 op/s; 1.7 MiB/s, 51 objects/s recovering 2024-04-25T04:25:40.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:40 smithi112 ceph-mon[50007]: osdmap e100: 8 total, 8 up, 8 in 2024-04-25T04:25:41.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:41 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1626 objects degraded (1.845%), 3 pgs degraded) 2024-04-25T04:25:41.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:41 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:25:41.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205465 10.99 MB/sec execute 325 sec latency 42.970 ms 2024-04-25T04:25:41.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:41 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1626 objects degraded (1.845%), 3 pgs degraded) 2024-04-25T04:25:41.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:41 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:25:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:42 smithi078 ceph-mon[62942]: pgmap v120: 97 pgs: 95 active+clean, 2 peering; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 586 op/s; 2.3 MiB/s, 54 keys/s, 43 objects/s recovering 2024-04-25T04:25:42.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205840 10.98 MB/sec execute 326 sec latency 41.751 ms 2024-04-25T04:25:42.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:42 smithi112 ceph-mon[50007]: pgmap v120: 97 pgs: 95 active+clean, 2 peering; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 586 op/s; 2.3 MiB/s, 54 keys/s, 43 objects/s recovering 2024-04-25T04:25:43.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206292 10.96 MB/sec execute 327 sec latency 50.499 ms 2024-04-25T04:25:44.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:44 smithi078 ceph-mon[62942]: pgmap v121: 97 pgs: 96 active+clean, 1 peering; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 547 op/s; 1.8 MiB/s, 42 keys/s, 33 objects/s recovering 2024-04-25T04:25:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:44 smithi112 ceph-mon[50007]: pgmap v121: 97 pgs: 96 active+clean, 1 peering; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 547 op/s; 1.8 MiB/s, 42 keys/s, 33 objects/s recovering 2024-04-25T04:25:44.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206648 10.96 MB/sec execute 328 sec latency 39.530 ms 2024-04-25T04:25:45.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207233 10.97 MB/sec execute 329 sec latency 61.003 ms 2024-04-25T04:25:46.406 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:46 smithi078 ceph-mon[62942]: pgmap v122: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.9 MiB/s wr, 723 op/s; 1.4 MiB/s, 33 keys/s, 22 objects/s recovering 2024-04-25T04:25:46.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:46 smithi112 ceph-mon[50007]: pgmap v122: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.9 MiB/s wr, 723 op/s; 1.4 MiB/s, 33 keys/s, 22 objects/s recovering 2024-04-25T04:25:46.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207698 10.97 MB/sec execute 330 sec latency 42.191 ms 2024-04-25T04:25:47.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208037 10.98 MB/sec execute 331 sec latency 45.582 ms 2024-04-25T04:25:48.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:48 smithi078 ceph-mon[62942]: pgmap v123: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.9 MiB/s wr, 365 op/s; 472 KiB/s, 29 keys/s, 2 objects/s recovering 2024-04-25T04:25:48.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:48 smithi112 ceph-mon[50007]: pgmap v123: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.9 MiB/s wr, 365 op/s; 472 KiB/s, 29 keys/s, 2 objects/s recovering 2024-04-25T04:25:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208605 11.01 MB/sec execute 332 sec latency 33.463 ms 2024-04-25T04:25:49.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208970 11.01 MB/sec execute 333 sec latency 39.967 ms 2024-04-25T04:25:50.247 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:50 smithi078 ceph-mon[62942]: pgmap v124: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 633 op/s; 410 KiB/s, 25 keys/s, 1 objects/s recovering 2024-04-25T04:25:50.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:50 smithi112 ceph-mon[50007]: pgmap v124: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 633 op/s; 410 KiB/s, 25 keys/s, 1 objects/s recovering 2024-04-25T04:25:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209189 10.99 MB/sec execute 334 sec latency 58.703 ms 2024-04-25T04:25:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209406 10.97 MB/sec execute 335 sec latency 72.704 ms 2024-04-25T04:25:52.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:52 smithi078 ceph-mon[62942]: pgmap v125: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.8 MiB/s wr, 547 op/s; 351 KiB/s, 21 keys/s, 1 objects/s recovering 2024-04-25T04:25:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:52 smithi112 ceph-mon[50007]: pgmap v125: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.8 MiB/s wr, 547 op/s; 351 KiB/s, 21 keys/s, 1 objects/s recovering 2024-04-25T04:25:52.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209651 10.94 MB/sec execute 336 sec latency 74.420 ms 2024-04-25T04:25:53.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209859 10.92 MB/sec execute 337 sec latency 55.399 ms 2024-04-25T04:25:54.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:54.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:54.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:54.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:54.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:54 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:54.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:54 smithi078 ceph-mon[62942]: Upgrade: osd.2 is safe to restart 2024-04-25T04:25:54.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:54 smithi078 ceph-mon[62942]: pgmap v126: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 562 op/s 2024-04-25T04:25:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:25:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:54 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-25T04:25:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:54 smithi112 ceph-mon[50007]: Upgrade: osd.2 is safe to restart 2024-04-25T04:25:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:54 smithi112 ceph-mon[50007]: pgmap v126: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 562 op/s 2024-04-25T04:25:54.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210120 10.92 MB/sec execute 338 sec latency 46.038 ms 2024-04-25T04:25:55.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210531 10.92 MB/sec execute 339 sec latency 82.343 ms 2024-04-25T04:25:56.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:56 smithi078 ceph-mon[62942]: pgmap v127: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 633 op/s 2024-04-25T04:25:56.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:56 smithi112 ceph-mon[50007]: pgmap v127: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 633 op/s 2024-04-25T04:25:56.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210931 10.90 MB/sec execute 340 sec latency 47.424 ms 2024-04-25T04:25:57.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:57 smithi078 ceph-mon[62942]: Upgrade: Updating osd.2 2024-04-25T04:25:57.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:57.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T04:25:57.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:25:57.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:57 smithi078 ceph-mon[62942]: Deploying daemon osd.2 on smithi078 2024-04-25T04:25:57.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211217 10.89 MB/sec execute 341 sec latency 53.035 ms 2024-04-25T04:25:57.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:57 smithi112 ceph-mon[50007]: Upgrade: Updating osd.2 2024-04-25T04:25:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:25:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-25T04:25:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:25:57.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:57 smithi112 ceph-mon[50007]: Deploying daemon osd.2 on smithi078 2024-04-25T04:25:58.655 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:58 smithi078 ceph-mon[62942]: pgmap v128: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 501 op/s 2024-04-25T04:25:58.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211542 10.90 MB/sec execute 342 sec latency 42.006 ms 2024-04-25T04:25:58.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:58 smithi112 ceph-mon[50007]: pgmap v128: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.8 MiB/s wr, 501 op/s 2024-04-25T04:25:59.186 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:25:59.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:25:59 smithi078 ceph-mon[62942]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:25:59.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211954 10.91 MB/sec execute 343 sec latency 37.882 ms 2024-04-25T04:25:59.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:25:59 smithi112 ceph-mon[50007]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 29s ago 14m 24.3M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (2m) 29s ago 14m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (2m) 115s ago 13m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 29s ago 14m 55.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (8m) 29s ago 8m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (8m) 115s ago 8m 14.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:26:00.188 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (4m) 29s ago 15m 507M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (3m) 115s ago 13m 450M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (3m) 29s ago 15m 47.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (3m) 115s ago 13m 35.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (7m) 29s ago 7m 280M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (7m) 115s ago 7m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 29s ago 14m 18.1M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (4m) 115s ago 13m 16.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (86s) 29s ago 12m 255M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (39s) 29s ago 12m 12.1M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 29s ago 11m 496M 1936M 17.2.0 e1d6a67b021e e26f580d5f8c 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 29s ago 11m 614M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (10m) 115s ago 10m 528M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (10m) 115s ago 10m 381M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (10m) 115s ago 10m 556M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (9m) 115s ago 9m 466M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:26:00.189 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 29s ago 13m 71.1M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:26:00.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:00 smithi078 ceph-mon[62942]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:00.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:00 smithi078 ceph-mon[62942]: pgmap v129: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 598 op/s 2024-04-25T04:26:00.771 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:26:00.771 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:26:00.771 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:26:00.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-25T04:26:00.772 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 6 2024-04-25T04:26:00.773 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:26:00.773 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:26:00.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212296 10.92 MB/sec execute 344 sec latency 57.507 ms 2024-04-25T04:26:00.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:00 smithi112 ceph-mon[50007]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:00.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:00 smithi112 ceph-mon[50007]: pgmap v129: 97 pgs: 97 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 598 op/s 2024-04-25T04:26:01.363 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:26:01.363 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:26:01.363 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:26:01.363 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:26:01.363 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:26:01.363 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:26:01.364 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:26:01.364 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:26:01.364 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:26:01.364 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "8/23 daemons upgraded", 2024-04-25T04:26:01.364 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:26:01.364 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:26:01.364 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:26:01.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:01 smithi078 ceph-mon[62942]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:01.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:01 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/395075853' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212522 10.91 MB/sec execute 345 sec latency 45.048 ms 2024-04-25T04:26:01.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:01 smithi112 ceph-mon[50007]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:01.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:01 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/395075853' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:01.959 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:26:02.508 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:02 smithi078 systemd[1]: Stopping Ceph osd.2 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:26:02.763 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:02 smithi078 ceph-mon[62942]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:02.763 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:02 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/380265925' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:26:02.763 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:02 smithi078 ceph-mon[62942]: pgmap v130: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 382 op/s 2024-04-25T04:26:02.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212678 10.88 MB/sec execute 346 sec latency 69.242 ms 2024-04-25T04:26:02.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:02 smithi112 ceph-mon[50007]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:02.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:02 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/380265925' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:26:02.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:02 smithi112 ceph-mon[50007]: pgmap v130: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 382 op/s 2024-04-25T04:26:03.014 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:02 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[40010]: 2024-04-25T04:26:02.790+0000 7f03f1bfa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:26:03.014 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:02 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[40010]: 2024-04-25T04:26:02.790+0000 7f03f1bfa700 -1 osd.2 100 *** Got signal Terminated *** 2024-04-25T04:26:03.014 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:02 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[40010]: 2024-04-25T04:26:02.790+0000 7f03f1bfa700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:26:03.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:03 smithi078 ceph-mon[62942]: osd.2 marked itself down and dead 2024-04-25T04:26:03.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212968 10.86 MB/sec execute 347 sec latency 68.599 ms 2024-04-25T04:26:03.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:03 smithi112 ceph-mon[50007]: osd.2 marked itself down and dead 2024-04-25T04:26:04.513 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:04 smithi078 podman[73977]: 2024-04-25 04:26:04.27275797 +0000 UTC m=+1.633602410 container died e26f580d5f8cccd9264ebdbde49df39a466a4f8867853c9466f264968b0ae080 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8, vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-25T04:26:04.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:04 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:26:04.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:04 smithi078 ceph-mon[62942]: osdmap e101: 8 total, 7 up, 8 in 2024-04-25T04:26:04.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:04 smithi078 ceph-mon[62942]: pgmap v132: 97 pgs: 8 peering, 89 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 518 op/s 2024-04-25T04:26:04.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213211 10.84 MB/sec execute 348 sec latency 851.249 ms 2024-04-25T04:26:04.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:04 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:26:04.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:04 smithi112 ceph-mon[50007]: osdmap e101: 8 total, 7 up, 8 in 2024-04-25T04:26:04.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:04 smithi112 ceph-mon[50007]: pgmap v132: 97 pgs: 8 peering, 89 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.8 MiB/s wr, 518 op/s 2024-04-25T04:26:05.027 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:05 smithi078 podman[73977]: 2024-04-25 04:26:05.019582413 +0000 UTC m=+2.380426865 container cleanup e26f580d5f8cccd9264ebdbde49df39a466a4f8867853c9466f264968b0ae080 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, release=754, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD) 2024-04-25T04:26:05.027 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:05 smithi078 bash[73977]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2 2024-04-25T04:26:05.540 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:05 smithi078 podman[73996]: 2024-04-25 04:26:05.313118249 +0000 UTC m=+1.028885788 container remove e26f580d5f8cccd9264ebdbde49df39a466a4f8867853c9466f264968b0ae080 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.expose-services=) 2024-04-25T04:26:05.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:05 smithi078 ceph-mon[62942]: osdmap e102: 8 total, 7 up, 8 in 2024-04-25T04:26:05.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213571 10.83 MB/sec execute 349 sec latency 48.233 ms 2024-04-25T04:26:05.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:05 smithi112 ceph-mon[50007]: osdmap e102: 8 total, 7 up, 8 in 2024-04-25T04:26:06.052 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:05 smithi078 podman[74084]: 2024-04-25 04:26:05.800328365 +0000 UTC m=+0.021579071 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:26:06.052 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:06 smithi078 podman[74084]: 2024-04-25 04:26:06.031431957 +0000 UTC m=+0.252682649 container create 7b727189752512a51b29dd4212a1268349b349cc0203a6f05eb671198ed363d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True) 2024-04-25T04:26:06.557 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:06 smithi078 podman[74084]: 2024-04-25 04:26:06.365941884 +0000 UTC m=+0.587192586 container init 7b727189752512a51b29dd4212a1268349b349cc0203a6f05eb671198ed363d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-25T04:26:06.557 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:06 smithi078 podman[74084]: 2024-04-25 04:26:06.371327261 +0000 UTC m=+0.592577950 container start 7b727189752512a51b29dd4212a1268349b349cc0203a6f05eb671198ed363d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-25T04:26:06.557 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:06 smithi078 podman[74084]: 2024-04-25 04:26:06.529611188 +0000 UTC m=+0.750861877 container attach 7b727189752512a51b29dd4212a1268349b349cc0203a6f05eb671198ed363d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-25T04:26:06.808 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:06 smithi078 podman[74084]: 2024-04-25 04:26:06.776953684 +0000 UTC m=+0.998204373 container died 7b727189752512a51b29dd4212a1268349b349cc0203a6f05eb671198ed363d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:26:06.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:06 smithi078 ceph-mon[62942]: pgmap v134: 97 pgs: 2 stale+active+clean, 18 peering, 77 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 474 op/s 2024-04-25T04:26:06.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213982 10.84 MB/sec execute 350 sec latency 56.486 ms 2024-04-25T04:26:06.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:06 smithi112 ceph-mon[50007]: pgmap v134: 97 pgs: 2 stale+active+clean, 18 peering, 77 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 474 op/s 2024-04-25T04:26:07.842 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:07 smithi078 ceph-mon[62942]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:26:07.843 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:07 smithi078 podman[74108]: 2024-04-25 04:26:07.727130545 +0000 UTC m=+0.936137438 container remove 7b727189752512a51b29dd4212a1268349b349cc0203a6f05eb671198ed363d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:26:07.843 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:07 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-25T04:26:07.843 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:07 smithi078 systemd[1]: Stopped Ceph osd.2 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:26:07.843 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:07 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.2.service: Consumed 2min 34.489s CPU time. 2024-04-25T04:26:07.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214474 10.83 MB/sec execute 351 sec latency 48.980 ms 2024-04-25T04:26:07.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:07 smithi112 ceph-mon[50007]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:26:08.102 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:08 smithi078 systemd[1]: Starting Ceph osd.2 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:26:08.360 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:08 smithi078 podman[74205]: 2024-04-25 04:26:08.195420683 +0000 UTC m=+0.025333391 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:26:08.360 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:08 smithi078 podman[74205]: 2024-04-25 04:26:08.328365046 +0000 UTC m=+0.158277733 container create 10a1417b4e47aa4e3455e6e4955b544e2e2a3618dd8da178ec516f8371d902ed (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-25T04:26:08.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214891 10.84 MB/sec execute 352 sec latency 42.277 ms 2024-04-25T04:26:08.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:08 smithi078 ceph-mon[62942]: pgmap v135: 97 pgs: 3 active+undersized, 3 active+undersized+degraded, 18 peering, 73 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.3 MiB/s wr, 323 op/s; 31/1674 objects degraded (1.852%) 2024-04-25T04:26:08.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:08.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:08.866 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:08 smithi078 podman[74205]: 2024-04-25 04:26:08.730938274 +0000 UTC m=+0.560850969 container init 10a1417b4e47aa4e3455e6e4955b544e2e2a3618dd8da178ec516f8371d902ed (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-25T04:26:08.866 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:08 smithi078 podman[74205]: 2024-04-25 04:26:08.736761794 +0000 UTC m=+0.566674478 container start 10a1417b4e47aa4e3455e6e4955b544e2e2a3618dd8da178ec516f8371d902ed (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:26:08.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:08 smithi112 ceph-mon[50007]: pgmap v135: 97 pgs: 3 active+undersized, 3 active+undersized+degraded, 18 peering, 73 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.3 MiB/s wr, 323 op/s; 31/1674 objects degraded (1.852%) 2024-04-25T04:26:08.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:08.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:09.120 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:08 smithi078 podman[74205]: 2024-04-25 04:26:08.891569371 +0000 UTC m=+0.721482071 container attach 10a1417b4e47aa4e3455e6e4955b544e2e2a3618dd8da178ec516f8371d902ed (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-25T04:26:09.375 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:09.375 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:09.375 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 bash[74205]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-25T04:26:09.376 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate[74220]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-25T04:26:09.627 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:09 smithi078 podman[74205]: 2024-04-25 04:26:09.384109017 +0000 UTC m=+1.214021718 container died 10a1417b4e47aa4e3455e6e4955b544e2e2a3618dd8da178ec516f8371d902ed (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-25T04:26:09.819 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:09 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 31/1674 objects degraded (1.852%), 3 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:09.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215413 10.86 MB/sec execute 353 sec latency 29.344 ms 2024-04-25T04:26:09.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:09 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 31/1674 objects degraded (1.852%), 3 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215873 10.88 MB/sec execute 354 sec latency 31.214 ms 2024-04-25T04:26:10.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:10 smithi078 ceph-mon[62942]: pgmap v136: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 600 op/s; 159/1686 objects degraded (9.431%) 2024-04-25T04:26:10.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:10 smithi112 ceph-mon[50007]: pgmap v136: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 600 op/s; 159/1686 objects degraded (9.431%) 2024-04-25T04:26:11.143 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 podman[74347]: 2024-04-25 04:26:11.026073597 +0000 UTC m=+1.628867255 container remove 10a1417b4e47aa4e3455e6e4955b544e2e2a3618dd8da178ec516f8371d902ed (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-25T04:26:11.395 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 podman[74402]: 2024-04-25 04:26:11.207934538 +0000 UTC m=+0.026373682 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:26:11.395 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 podman[74402]: 2024-04-25 04:26:11.336051307 +0000 UTC m=+0.154490449 container create ee973230cd51542cb98c14eb3538fd2b4c899b3d8ac0c32cd49d11ec37c1fa21 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:26:11.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216182 10.88 MB/sec execute 355 sec latency 34.431 ms 2024-04-25T04:26:11.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:11 smithi078 ceph-mon[62942]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-25T04:26:11.901 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 podman[74402]: 2024-04-25 04:26:11.647403931 +0000 UTC m=+0.465843067 container init ee973230cd51542cb98c14eb3538fd2b4c899b3d8ac0c32cd49d11ec37c1fa21 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0) 2024-04-25T04:26:11.901 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 podman[74402]: 2024-04-25 04:26:11.657606678 +0000 UTC m=+0.476045814 container start ee973230cd51542cb98c14eb3538fd2b4c899b3d8ac0c32cd49d11ec37c1fa21 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True) 2024-04-25T04:26:11.901 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 ceph-osd[74452]: -- 172.21.15.78:0/1407733682 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557276ed9680 con 0x557276ea3000 2024-04-25T04:26:11.901 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 bash[74402]: ee973230cd51542cb98c14eb3538fd2b4c899b3d8ac0c32cd49d11ec37c1fa21 2024-04-25T04:26:11.901 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:11 smithi078 systemd[1]: Started Ceph osd.2 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:26:11.902 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:11 smithi112 ceph-mon[50007]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-25T04:26:12.403 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:12 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[74448]: 2024-04-25T04:26:12.250+0000 7fe05c6af7c0 -1 Falling back to public interface 2024-04-25T04:26:12.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216494 10.86 MB/sec execute 356 sec latency 35.200 ms 2024-04-25T04:26:13.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:13.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:13.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:26:13.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:12 smithi078 ceph-mon[62942]: pgmap v137: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 585 op/s; 160/1698 objects degraded (9.423%) 2024-04-25T04:26:13.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:13.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:13.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:26:13.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:12 smithi112 ceph-mon[50007]: pgmap v137: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 585 op/s; 160/1698 objects degraded (9.423%) 2024-04-25T04:26:13.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216860 10.84 MB/sec execute 357 sec latency 43.567 ms 2024-04-25T04:26:14.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:14 smithi078 ceph-mon[62942]: pgmap v138: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 5.3 MiB/s wr, 530 op/s; 165/1722 objects degraded (9.582%) 2024-04-25T04:26:14.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217199 10.83 MB/sec execute 358 sec latency 32.413 ms 2024-04-25T04:26:14.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:14 smithi112 ceph-mon[50007]: pgmap v138: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 5.3 MiB/s wr, 530 op/s; 165/1722 objects degraded (9.582%) 2024-04-25T04:26:15.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:15 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 165/1722 objects degraded (9.582%), 19 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:15.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217617 10.84 MB/sec execute 359 sec latency 48.186 ms 2024-04-25T04:26:15.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:15 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 165/1722 objects degraded (9.582%), 19 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:16.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:16 smithi078 ceph-mon[62942]: pgmap v139: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 167/1746 objects degraded (9.565%) 2024-04-25T04:26:16.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218120 10.83 MB/sec execute 360 sec latency 42.743 ms 2024-04-25T04:26:16.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:16 smithi112 ceph-mon[50007]: pgmap v139: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 167/1746 objects degraded (9.565%) 2024-04-25T04:26:17.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218501 10.84 MB/sec execute 361 sec latency 39.593 ms 2024-04-25T04:26:18.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:18 smithi078 ceph-mon[62942]: pgmap v140: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 490 op/s; 167/1740 objects degraded (9.598%) 2024-04-25T04:26:18.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:18 smithi112 ceph-mon[50007]: pgmap v140: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 490 op/s; 167/1740 objects degraded (9.598%) 2024-04-25T04:26:18.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218911 10.86 MB/sec execute 362 sec latency 38.701 ms 2024-04-25T04:26:18.999 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:18 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[74448]: 2024-04-25T04:26:18.738+0000 7fe05c6af7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-25T04:26:19.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:19.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:19.757 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:19 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[74448]: 2024-04-25T04:26:19.732+0000 7fe05c6af7c0 -1 osd.2 100 log_to_monitors true 2024-04-25T04:26:19.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219395 10.87 MB/sec execute 363 sec latency 36.709 ms 2024-04-25T04:26:20.511 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:20 smithi078 ceph-mon[62942]: from='osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T04:26:20.511 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:20 smithi078 ceph-mon[62942]: pgmap v141: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 719 op/s; 164/1716 objects degraded (9.557%) 2024-04-25T04:26:20.511 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:26:20 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[74448]: 2024-04-25T04:26:20.295+0000 7fe04db07700 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:26:20.551 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:20 smithi112 ceph-mon[50007]: from='osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-25T04:26:20.551 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:20 smithi112 ceph-mon[50007]: pgmap v141: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 719 op/s; 164/1716 objects degraded (9.557%) 2024-04-25T04:26:20.854 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219838 10.87 MB/sec execute 364 sec latency 29.021 ms 2024-04-25T04:26:21.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:21 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 164/1716 objects degraded (9.557%), 19 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:21.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:21 smithi078 ceph-mon[62942]: from='osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-25T04:26:21.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:21 smithi078 ceph-mon[62942]: osdmap e103: 8 total, 7 up, 8 in 2024-04-25T04:26:21.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:21 smithi078 ceph-mon[62942]: from='osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:26:21.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:21 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 164/1716 objects degraded (9.557%), 19 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:21 smithi112 ceph-mon[50007]: from='osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-25T04:26:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:21 smithi112 ceph-mon[50007]: osdmap e103: 8 total, 7 up, 8 in 2024-04-25T04:26:21.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:21 smithi112 ceph-mon[50007]: from='osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:26:21.854 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220143 10.85 MB/sec execute 365 sec latency 561.163 ms 2024-04-25T04:26:22.532 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:22 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:26:22.532 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:22 smithi078 ceph-mon[62942]: osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397] boot 2024-04-25T04:26:22.532 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:22 smithi078 ceph-mon[62942]: osdmap e104: 8 total, 8 up, 8 in 2024-04-25T04:26:22.532 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:26:22.532 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:22 smithi078 ceph-mon[62942]: pgmap v144: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 547 op/s; 166/1719 objects degraded (9.657%) 2024-04-25T04:26:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:22 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:26:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:22 smithi112 ceph-mon[50007]: osd.2 [v2:172.21.15.78:6818/2331296397,v1:172.21.15.78:6819/2331296397] boot 2024-04-25T04:26:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:22 smithi112 ceph-mon[50007]: osdmap e104: 8 total, 8 up, 8 in 2024-04-25T04:26:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-25T04:26:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:22 smithi112 ceph-mon[50007]: pgmap v144: 97 pgs: 8 active+undersized, 19 active+undersized+degraded, 70 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 547 op/s; 166/1719 objects degraded (9.657%) 2024-04-25T04:26:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220185 10.83 MB/sec execute 366 sec latency 1561.192 ms 2024-04-25T04:26:23.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:23 smithi078 ceph-mon[62942]: osdmap e105: 8 total, 8 up, 8 in 2024-04-25T04:26:23.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:23.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:23.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:23.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:23.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:23 smithi112 ceph-mon[50007]: osdmap e105: 8 total, 8 up, 8 in 2024-04-25T04:26:23.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:23.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:23.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:23.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:23.854 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220201 10.80 MB/sec execute 367 sec latency 2461.856 ms 2024-04-25T04:26:24.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:24 smithi078 ceph-mon[62942]: osdmap e106: 8 total, 8 up, 8 in 2024-04-25T04:26:24.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:24.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:24.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:24 smithi078 ceph-mon[62942]: pgmap v147: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 11 active+undersized+degraded, 75 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s wr, 416 op/s; 141/1590 objects degraded (8.868%); 2.0 MiB/s, 3 objects/s recovering 2024-04-25T04:26:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:24 smithi112 ceph-mon[50007]: osdmap e106: 8 total, 8 up, 8 in 2024-04-25T04:26:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:24 smithi112 ceph-mon[50007]: pgmap v147: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 11 active+undersized+degraded, 75 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s wr, 416 op/s; 141/1590 objects degraded (8.868%); 2.0 MiB/s, 3 objects/s recovering 2024-04-25T04:26:24.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220476 10.78 MB/sec execute 368 sec latency 184.577 ms 2024-04-25T04:26:25.800 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:25 smithi078 ceph-mon[62942]: osdmap e107: 8 total, 8 up, 8 in 2024-04-25T04:26:25.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220829 10.77 MB/sec execute 369 sec latency 1003.107 ms 2024-04-25T04:26:25.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:25 smithi112 ceph-mon[50007]: osdmap e107: 8 total, 8 up, 8 in 2024-04-25T04:26:26.766 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:26 smithi078 ceph-mon[62942]: osdmap e108: 8 total, 8 up, 8 in 2024-04-25T04:26:26.766 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:26 smithi078 ceph-mon[62942]: pgmap v150: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 active+undersized+degraded, 84 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.4 MiB/s wr, 477 op/s; 111/1437 objects degraded (7.724%); 4.0 MiB/s, 10 objects/s recovering 2024-04-25T04:26:26.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220905 10.74 MB/sec execute 370 sec latency 1041.951 ms 2024-04-25T04:26:27.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:26 smithi112 ceph-mon[50007]: osdmap e108: 8 total, 8 up, 8 in 2024-04-25T04:26:27.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:26 smithi112 ceph-mon[50007]: pgmap v150: 97 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 2 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 active+undersized+degraded, 84 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 4.4 MiB/s wr, 477 op/s; 111/1437 objects degraded (7.724%); 4.0 MiB/s, 10 objects/s recovering 2024-04-25T04:26:27.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:27 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 111/1437 objects degraded (7.724%), 6 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:27.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:27 smithi078 ceph-mon[62942]: osdmap e109: 8 total, 8 up, 8 in 2024-04-25T04:26:27.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221289 10.74 MB/sec execute 371 sec latency 52.668 ms 2024-04-25T04:26:28.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:27 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 111/1437 objects degraded (7.724%), 6 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:28.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:27 smithi112 ceph-mon[50007]: osdmap e109: 8 total, 8 up, 8 in 2024-04-25T04:26:28.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:28 smithi078 ceph-mon[62942]: pgmap v152: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 87 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 MiB/s wr, 387 op/s; 84/1422 objects degraded (5.907%); 3.4 MiB/s, 18 objects/s recovering 2024-04-25T04:26:28.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221765 10.75 MB/sec execute 372 sec latency 36.337 ms 2024-04-25T04:26:29.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:28 smithi112 ceph-mon[50007]: pgmap v152: 97 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 87 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 MiB/s wr, 387 op/s; 84/1422 objects degraded (5.907%); 3.4 MiB/s, 18 objects/s recovering 2024-04-25T04:26:29.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222160 10.75 MB/sec execute 373 sec latency 34.542 ms 2024-04-25T04:26:30.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:26:30.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:26:30.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:26:30.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:30.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:30.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:30.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:30.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:30.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:30.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222611 10.78 MB/sec execute 374 sec latency 40.431 ms 2024-04-25T04:26:31.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:31 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:31.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:31 smithi078 ceph-mon[62942]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-25T04:26:31.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:31 smithi078 ceph-mon[62942]: pgmap v153: 97 pgs: 1 peering, 96 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.7 MiB/s wr, 426 op/s; 1.5 MiB/s, 34 objects/s recovering 2024-04-25T04:26:31.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:31 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:31.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:31 smithi112 ceph-mon[50007]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-25T04:26:31.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:31 smithi112 ceph-mon[50007]: pgmap v153: 97 pgs: 1 peering, 96 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.7 MiB/s wr, 426 op/s; 1.5 MiB/s, 34 objects/s recovering 2024-04-25T04:26:31.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223028 10.79 MB/sec execute 375 sec latency 28.503 ms 2024-04-25T04:26:32.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:32 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 84/1422 objects degraded (5.907%), 4 pgs degraded) 2024-04-25T04:26:32.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:32 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:26:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:32 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 84/1422 objects degraded (5.907%), 4 pgs degraded) 2024-04-25T04:26:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:32 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:26:32.508 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:26:32.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223375 10.78 MB/sec execute 376 sec latency 41.038 ms 2024-04-25T04:26:33.134 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:33 smithi078 ceph-mon[62942]: pgmap v154: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.8 MiB/s wr, 313 op/s; 107 KiB/s, 23 objects/s recovering 2024-04-25T04:26:33.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:33 smithi112 ceph-mon[50007]: pgmap v154: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.8 MiB/s wr, 313 op/s; 107 KiB/s, 23 objects/s recovering 2024-04-25T04:26:33.411 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 10s ago 14m 24.3M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (2m) 10s ago 14m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (2m) 2m ago 14m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 10s ago 14m 54.9M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (8m) 10s ago 8m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (8m) 2m ago 8m 14.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (5m) 10s ago 16m 508M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 10s ago 16m 48.7M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (3m) 2m ago 14m 35.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (8m) 10s ago 8m 276M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (8m) 2m ago 8m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:26:33.412 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 10s ago 14m 17.5M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (5m) 2m ago 13m 16.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (119s) 10s ago 13m 302M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (72s) 10s ago 12m 228M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (22s) 10s ago 12m 12.1M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 10s ago 11m 394M 1936M 17.2.0 e1d6a67b021e ec53e2c1476f 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (11m) 2m ago 11m 528M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (11m) 2m ago 11m 381M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (10m) 2m ago 10m 556M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (10m) 2m ago 10m 466M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:26:33.413 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 10s ago 13m 71.1M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:26:33.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223583 10.75 MB/sec execute 377 sec latency 63.847 ms 2024-04-25T04:26:33.885 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:26:33.885 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:26:33.885 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:26:33.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:33.885 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 3 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 7 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:26:33.886 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:26:34.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:34 smithi078 ceph-mon[62942]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:34.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:34 smithi078 ceph-mon[62942]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:34.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:34 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3846718535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:34.277 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:26:34.277 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:26:34.277 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:26:34.277 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:26:34.277 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:26:34.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:26:34.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:26:34.278 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:26:34.278 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:26:34.278 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "9/23 daemons upgraded", 2024-04-25T04:26:34.278 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:26:34.278 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:26:34.278 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:26:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:34 smithi112 ceph-mon[50007]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:34 smithi112 ceph-mon[50007]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:34 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3846718535' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:26:34.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223746 10.73 MB/sec execute 378 sec latency 71.171 ms 2024-04-25T04:26:34.985 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:26:35.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:35 smithi078 ceph-mon[62942]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:35.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:35 smithi078 ceph-mon[62942]: pgmap v155: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 4.9 MiB/s wr, 506 op/s; 81 KiB/s, 20 objects/s recovering 2024-04-25T04:26:35.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:35 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/4208254459' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:26:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:35 smithi112 ceph-mon[50007]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:35 smithi112 ceph-mon[50007]: pgmap v155: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 4.9 MiB/s wr, 506 op/s; 81 KiB/s, 20 objects/s recovering 2024-04-25T04:26:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:35 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/4208254459' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:26:35.859 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223992 10.72 MB/sec execute 379 sec latency 64.914 ms 2024-04-25T04:26:36.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:36 smithi078 ceph-mon[62942]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:36.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:36 smithi112 ceph-mon[50007]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:26:36.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224209 10.71 MB/sec execute 380 sec latency 107.685 ms 2024-04-25T04:26:37.118 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:37 smithi078 ceph-mon[62942]: pgmap v156: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 484 op/s; 69 KiB/s, 17 objects/s recovering 2024-04-25T04:26:37.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:37 smithi112 ceph-mon[50007]: pgmap v156: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 484 op/s; 69 KiB/s, 17 objects/s recovering 2024-04-25T04:26:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224535 10.69 MB/sec execute 381 sec latency 94.091 ms 2024-04-25T04:26:38.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:38 smithi078 ceph-mon[62942]: pgmap v157: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 424 op/s; 7.7 KiB/s, 11 objects/s recovering 2024-04-25T04:26:38.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:38.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:38.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:38 smithi112 ceph-mon[50007]: pgmap v157: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 424 op/s; 7.7 KiB/s, 11 objects/s recovering 2024-04-25T04:26:38.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:38.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:38.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224957 10.69 MB/sec execute 382 sec latency 43.902 ms 2024-04-25T04:26:39.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:39.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:39.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225466 10.71 MB/sec execute 383 sec latency 43.076 ms 2024-04-25T04:26:40.447 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:40 smithi078 ceph-mon[62942]: pgmap v158: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.3 MiB/s wr, 532 op/s; 7.4 KiB/s, 10 objects/s recovering 2024-04-25T04:26:40.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:40 smithi112 ceph-mon[50007]: pgmap v158: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.3 MiB/s wr, 532 op/s; 7.4 KiB/s, 10 objects/s recovering 2024-04-25T04:26:40.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225952 10.73 MB/sec execute 384 sec latency 31.545 ms 2024-04-25T04:26:41.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226406 10.75 MB/sec execute 385 sec latency 32.249 ms 2024-04-25T04:26:42.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:42 smithi078 ceph-mon[62942]: pgmap v159: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 469 op/s 2024-04-25T04:26:42.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:42 smithi112 ceph-mon[50007]: pgmap v159: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 469 op/s 2024-04-25T04:26:42.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226819 10.74 MB/sec execute 386 sec latency 33.717 ms 2024-04-25T04:26:43.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227139 10.72 MB/sec execute 387 sec latency 36.089 ms 2024-04-25T04:26:44.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:44 smithi078 ceph-mon[62942]: pgmap v160: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.0 MiB/s wr, 585 op/s 2024-04-25T04:26:44.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:44 smithi112 ceph-mon[50007]: pgmap v160: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.0 MiB/s wr, 585 op/s 2024-04-25T04:26:44.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227453 10.72 MB/sec execute 388 sec latency 46.272 ms 2024-04-25T04:26:45.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:45.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:45 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:45.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:45 smithi078 ceph-mon[62942]: Upgrade: osd.3 is safe to restart 2024-04-25T04:26:45.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:45.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:45 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-25T04:26:45.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:45 smithi112 ceph-mon[50007]: Upgrade: osd.3 is safe to restart 2024-04-25T04:26:45.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227795 10.71 MB/sec execute 389 sec latency 84.707 ms 2024-04-25T04:26:46.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:46 smithi078 ceph-mon[62942]: pgmap v161: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 450 op/s 2024-04-25T04:26:46.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:46 smithi112 ceph-mon[50007]: pgmap v161: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 450 op/s 2024-04-25T04:26:46.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228116 10.69 MB/sec execute 390 sec latency 49.314 ms 2024-04-25T04:26:47.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228325 10.68 MB/sec execute 391 sec latency 79.313 ms 2024-04-25T04:26:48.085 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:47 smithi078 ceph-mon[62942]: Upgrade: Updating osd.3 2024-04-25T04:26:48.085 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:47 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:48.085 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:47 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T04:26:48.085 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:47 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:26:48.085 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:47 smithi078 ceph-mon[62942]: Deploying daemon osd.3 on smithi078 2024-04-25T04:26:48.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:47 smithi112 ceph-mon[50007]: Upgrade: Updating osd.3 2024-04-25T04:26:48.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:47 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:26:48.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:47 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-25T04:26:48.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:47 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:26:48.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:47 smithi112 ceph-mon[50007]: Deploying daemon osd.3 on smithi078 2024-04-25T04:26:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228538 10.66 MB/sec execute 392 sec latency 59.395 ms 2024-04-25T04:26:48.857 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:48 smithi078 ceph-mon[62942]: pgmap v162: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 498 op/s 2024-04-25T04:26:49.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:48 smithi112 ceph-mon[50007]: pgmap v162: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 498 op/s 2024-04-25T04:26:49.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228861 10.66 MB/sec execute 393 sec latency 94.019 ms 2024-04-25T04:26:50.441 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:50 smithi078 ceph-mon[62942]: pgmap v163: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.6 MiB/s wr, 616 op/s 2024-04-25T04:26:50.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:50 smithi112 ceph-mon[50007]: pgmap v163: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.6 MiB/s wr, 616 op/s 2024-04-25T04:26:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229115 10.65 MB/sec execute 394 sec latency 45.049 ms 2024-04-25T04:26:51.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229445 10.66 MB/sec execute 395 sec latency 37.806 ms 2024-04-25T04:26:52.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:52 smithi078 ceph-mon[62942]: pgmap v164: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-04-25T04:26:52.459 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:52 smithi078 systemd[1]: Stopping Ceph osd.3 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:26:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:52 smithi112 ceph-mon[50007]: pgmap v164: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-04-25T04:26:52.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229776 10.67 MB/sec execute 396 sec latency 39.868 ms 2024-04-25T04:26:52.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:52 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[43279]: 2024-04-25T04:26:52.733+0000 7f013de3f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:26:52.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:52 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[43279]: 2024-04-25T04:26:52.733+0000 7f013de3f700 -1 osd.3 109 *** Got signal Terminated *** 2024-04-25T04:26:52.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:52 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[43279]: 2024-04-25T04:26:52.733+0000 7f013de3f700 -1 osd.3 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:26:53.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:53 smithi078 ceph-mon[62942]: osd.3 marked itself down and dead 2024-04-25T04:26:53.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:53 smithi112 ceph-mon[50007]: osd.3 marked itself down and dead 2024-04-25T04:26:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:26:53.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229872 10.65 MB/sec execute 397 sec latency 632.832 ms 2024-04-25T04:26:54.205 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:53 smithi078 podman[76567]: 2024-04-25 04:26:53.893275523 +0000 UTC m=+1.293199808 container died ec53e2c1476f4527971dc2254eea5d12b51f80e55411cc390fdd5bd186ba5942 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, architecture=x86_64, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, release=754, GIT_BRANCH=HEAD) 2024-04-25T04:26:54.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:54 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:26:54.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:54 smithi078 ceph-mon[62942]: osdmap e110: 8 total, 7 up, 8 in 2024-04-25T04:26:54.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:54 smithi078 ceph-mon[62942]: pgmap v166: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-04-25T04:26:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:54 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:26:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:54 smithi112 ceph-mon[50007]: osdmap e110: 8 total, 7 up, 8 in 2024-04-25T04:26:54.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:54 smithi112 ceph-mon[50007]: pgmap v166: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-04-25T04:26:54.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230081 10.64 MB/sec execute 398 sec latency 1039.366 ms 2024-04-25T04:26:54.962 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:54 smithi078 podman[76567]: 2024-04-25 04:26:54.799663841 +0000 UTC m=+2.199588130 container cleanup ec53e2c1476f4527971dc2254eea5d12b51f80e55411cc390fdd5bd186ba5942 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.19.8, distribution-scope=public, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, release=754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True) 2024-04-25T04:26:54.962 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:54 smithi078 bash[76567]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3 2024-04-25T04:26:55.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:55 smithi078 ceph-mon[62942]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:26:55.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:55 smithi078 ceph-mon[62942]: osdmap e111: 8 total, 7 up, 8 in 2024-04-25T04:26:55.464 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:55 smithi078 podman[76583]: 2024-04-25 04:26:55.234168091 +0000 UTC m=+1.337320059 container remove ec53e2c1476f4527971dc2254eea5d12b51f80e55411cc390fdd5bd186ba5942 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, ceph=True, name=centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-25T04:26:55.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:55 smithi112 ceph-mon[50007]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:26:55.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:55 smithi112 ceph-mon[50007]: osdmap e111: 8 total, 7 up, 8 in 2024-04-25T04:26:55.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230400 10.63 MB/sec execute 399 sec latency 32.637 ms 2024-04-25T04:26:55.966 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:55 smithi078 podman[76666]: 2024-04-25 04:26:55.73623009 +0000 UTC m=+0.024719985 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:26:55.966 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:55 smithi078 podman[76666]: 2024-04-25 04:26:55.877311882 +0000 UTC m=+0.165801747 container create 7a86485d7ad1f5dd67a271376fc8c70b8c62d55f956d5b32251ac5bd0af46ada (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-25T04:26:56.219 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:56 smithi078 podman[76666]: 2024-04-25 04:26:56.193662493 +0000 UTC m=+0.482152357 container init 7a86485d7ad1f5dd67a271376fc8c70b8c62d55f956d5b32251ac5bd0af46ada (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:26:56.219 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:56 smithi078 podman[76666]: 2024-04-25 04:26:56.202097846 +0000 UTC m=+0.490587706 container start 7a86485d7ad1f5dd67a271376fc8c70b8c62d55f956d5b32251ac5bd0af46ada (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:26:56.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:56 smithi078 ceph-mon[62942]: pgmap v168: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 18 peering, 7 stale+active+clean, 68 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 16/1671 objects degraded (0.958%) 2024-04-25T04:26:56.470 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:56 smithi078 podman[76666]: 2024-04-25 04:26:56.384390117 +0000 UTC m=+0.672879983 container attach 7a86485d7ad1f5dd67a271376fc8c70b8c62d55f956d5b32251ac5bd0af46ada (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-25T04:26:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:56 smithi112 ceph-mon[50007]: pgmap v168: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 18 peering, 7 stale+active+clean, 68 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 16/1671 objects degraded (0.958%) 2024-04-25T04:26:56.722 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:56 smithi078 podman[76666]: 2024-04-25 04:26:56.651620611 +0000 UTC m=+0.940110485 container died 7a86485d7ad1f5dd67a271376fc8c70b8c62d55f956d5b32251ac5bd0af46ada (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-25T04:26:56.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230676 10.61 MB/sec execute 400 sec latency 39.556 ms 2024-04-25T04:26:57.488 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:57 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 16/1671 objects degraded (0.958%), 3 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:57 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 16/1671 objects degraded (0.958%), 3 pgs degraded (PG_DEGRADED) 2024-04-25T04:26:57.753 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:57 smithi078 podman[76691]: 2024-04-25 04:26:57.737278044 +0000 UTC m=+1.079255905 container remove 7a86485d7ad1f5dd67a271376fc8c70b8c62d55f956d5b32251ac5bd0af46ada (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-25T04:26:57.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230917 10.60 MB/sec execute 401 sec latency 69.932 ms 2024-04-25T04:26:58.002 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:57 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-25T04:26:58.002 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:57 smithi078 systemd[1]: Stopped Ceph osd.3 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:26:58.002 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:57 smithi078 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.3.service: Consumed 4min 26.951s CPU time. 2024-04-25T04:26:58.253 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:58 smithi078 systemd[1]: Starting Ceph osd.3 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:26:58.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:26:58 smithi078 ceph-mon[62942]: pgmap v169: 97 pgs: 5 active+undersized, 9 active+undersized+degraded, 18 peering, 3 stale+active+clean, 62 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 224 op/s; 50/1692 objects degraded (2.955%) 2024-04-25T04:26:58.519 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:58 smithi078 podman[76789]: 2024-04-25 04:26:58.265797056 +0000 UTC m=+0.031769286 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:26:58.519 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:58 smithi078 podman[76789]: 2024-04-25 04:26:58.393805127 +0000 UTC m=+0.159777351 container create ec8ba043096d0549f0a887ae316931442849ad475ab07ebab2e38933e38181dc (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-25T04:26:58.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:26:58 smithi112 ceph-mon[50007]: pgmap v169: 97 pgs: 5 active+undersized, 9 active+undersized+degraded, 18 peering, 3 stale+active+clean, 62 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 224 op/s; 50/1692 objects degraded (2.955%) 2024-04-25T04:26:58.769 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:58 smithi078 podman[76789]: 2024-04-25 04:26:58.672411382 +0000 UTC m=+0.438383625 container init ec8ba043096d0549f0a887ae316931442849ad475ab07ebab2e38933e38181dc (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-25T04:26:58.769 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:58 smithi078 podman[76789]: 2024-04-25 04:26:58.680348383 +0000 UTC m=+0.446320609 container start ec8ba043096d0549f0a887ae316931442849ad475ab07ebab2e38933e38181dc (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:26:58.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231263 10.60 MB/sec execute 402 sec latency 49.204 ms 2024-04-25T04:26:59.020 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:58 smithi078 podman[76789]: 2024-04-25 04:26:58.789064957 +0000 UTC m=+0.555037176 container attach ec8ba043096d0549f0a887ae316931442849ad475ab07ebab2e38933e38181dc (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-25T04:26:59.529 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:59.529 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:59.529 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-25T04:26:59.529 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-25T04:26:59.530 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-25T04:26:59.530 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-25T04:26:59.530 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-25T04:26:59.530 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-25T04:26:59.531 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate[76801]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-25T04:26:59.531 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:59.531 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:26:59.531 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-25T04:26:59.531 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-25T04:26:59.532 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-25T04:26:59.532 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-25T04:26:59.532 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-25T04:26:59.532 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-25T04:26:59.532 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 bash[76789]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-25T04:26:59.532 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:26:59 smithi078 podman[76789]: 2024-04-25 04:26:59.396186642 +0000 UTC m=+1.162158862 container died ec8ba043096d0549f0a887ae316931442849ad475ab07ebab2e38933e38181dc (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-25T04:26:59.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231753 10.59 MB/sec execute 403 sec latency 37.156 ms 2024-04-25T04:27:00.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:00 smithi078 ceph-mon[62942]: pgmap v170: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.5 MiB/s wr, 332 op/s; 220/1710 objects degraded (12.865%) 2024-04-25T04:27:00.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:00 smithi078 ceph-mon[62942]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-25T04:27:00.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:00 smithi112 ceph-mon[50007]: pgmap v170: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.5 MiB/s wr, 332 op/s; 220/1710 objects degraded (12.865%) 2024-04-25T04:27:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:00 smithi112 ceph-mon[50007]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-25T04:27:00.828 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:00 smithi078 podman[76931]: 2024-04-25 04:27:00.708375278 +0000 UTC m=+1.300150337 container remove ec8ba043096d0549f0a887ae316931442849ad475ab07ebab2e38933e38181dc (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-04-25T04:27:00.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232091 10.58 MB/sec execute 404 sec latency 38.318 ms 2024-04-25T04:27:01.079 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:00 smithi078 podman[76984]: 2024-04-25 04:27:00.863218155 +0000 UTC m=+0.023062371 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:27:01.079 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:00 smithi078 podman[76984]: 2024-04-25 04:27:00.976335263 +0000 UTC m=+0.136179443 container create a9bbda282776b2362a262b861649957f1db4d448043dde132c185d94c4177343 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-25T04:27:01.338 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:01 smithi078 podman[76984]: 2024-04-25 04:27:01.237948559 +0000 UTC m=+0.397792757 container init a9bbda282776b2362a262b861649957f1db4d448043dde132c185d94c4177343 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:27:01.338 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:01 smithi078 podman[76984]: 2024-04-25 04:27:01.247201931 +0000 UTC m=+0.407046117 container start a9bbda282776b2362a262b861649957f1db4d448043dde132c185d94c4177343 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:27:01.338 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:01 smithi078 ceph-osd[77002]: -- 172.21.15.78:0/310831840 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55dc95ce1680 con 0x55dc95caac00 2024-04-25T04:27:01.598 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:01 smithi078 bash[76984]: a9bbda282776b2362a262b861649957f1db4d448043dde132c185d94c4177343 2024-04-25T04:27:01.598 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:01 smithi078 systemd[1]: Started Ceph osd.3 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:27:01.849 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:01 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[76998]: 2024-04-25T04:27:01.827+0000 7f209c3037c0 -1 Falling back to public interface 2024-04-25T04:27:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232516 10.60 MB/sec execute 405 sec latency 43.346 ms 2024-04-25T04:27:02.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:02.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:02.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:02.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:02 smithi078 ceph-mon[62942]: pgmap v171: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.9 MiB/s wr, 539 op/s; 216/1704 objects degraded (12.676%) 2024-04-25T04:27:02.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232946 10.60 MB/sec execute 406 sec latency 34.058 ms 2024-04-25T04:27:02.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:02.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:02.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:02.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:02 smithi112 ceph-mon[50007]: pgmap v171: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.9 MiB/s wr, 539 op/s; 216/1704 objects degraded (12.676%) 2024-04-25T04:27:03.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:03 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 216/1704 objects degraded (12.676%), 26 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:03.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233335 10.62 MB/sec execute 407 sec latency 54.590 ms 2024-04-25T04:27:03.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:03 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 216/1704 objects degraded (12.676%), 26 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:04.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:04 smithi078 ceph-mon[62942]: pgmap v172: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.7 MiB/s wr, 463 op/s; 216/1698 objects degraded (12.721%) 2024-04-25T04:27:04.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233607 10.61 MB/sec execute 408 sec latency 52.268 ms 2024-04-25T04:27:04.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:04 smithi112 ceph-mon[50007]: pgmap v172: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.7 MiB/s wr, 463 op/s; 216/1698 objects degraded (12.721%) 2024-04-25T04:27:05.527 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:27:05.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233870 10.60 MB/sec execute 409 sec latency 39.235 ms 2024-04-25T04:27:06.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:06 smithi078 ceph-mon[62942]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:06.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:06 smithi078 ceph-mon[62942]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:06.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:06 smithi078 ceph-mon[62942]: pgmap v173: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 354 op/s; 214/1689 objects degraded (12.670%) 2024-04-25T04:27:06.511 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:27:06.511 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 44s ago 15m 24.3M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:27:06.511 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (3m) 44s ago 15m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:27:06.511 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (3m) 3m ago 14m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:27:06.511 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 44s ago 15m 54.9M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:27:06.511 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (9m) 44s ago 9m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:27:06.511 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (9m) 3m ago 9m 14.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (5m) 44s ago 16m 508M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (4m) 3m ago 14m 450M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 44s ago 16m 48.7M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (4m) 3m ago 14m 35.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (9m) 44s ago 9m 276M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (8m) 3m ago 8m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 44s ago 15m 17.5M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (6m) 3m ago 14m 16.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 44s ago 13m 302M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (105s) 44s ago 13m 228M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (55s) 44s ago 12m 12.1M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 starting - - - 1936M 2024-04-25T04:27:06.512 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (12m) 3m ago 12m 528M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:27:06.513 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (11m) 3m ago 11m 381M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:27:06.513 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (11m) 3m ago 11m 556M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:27:06.513 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (10m) 3m ago 10m 466M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:27:06.513 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 44s ago 14m 71.1M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:27:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:06 smithi112 ceph-mon[50007]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:06 smithi112 ceph-mon[50007]: from='client.44257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:06.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:06 smithi112 ceph-mon[50007]: pgmap v173: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 354 op/s; 214/1689 objects degraded (12.670%) 2024-04-25T04:27:06.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234104 10.58 MB/sec execute 410 sec latency 53.287 ms 2024-04-25T04:27:07.090 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 3 2024-04-25T04:27:07.091 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 7 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:27:07.092 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:27:07.458 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:07 smithi078 ceph-mon[62942]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:07.458 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:07 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2466917186' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:07.573 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:27:07.573 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:27:07.573 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:27:07.573 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:27:07.573 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:27:07.573 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:27:07.573 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:27:07.574 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:27:07.574 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:27:07.574 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "9/23 daemons upgraded", 2024-04-25T04:27:07.574 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:27:07.574 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:27:07.574 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:27:07.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:07 smithi112 ceph-mon[50007]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:07.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:07 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2466917186' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:07.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234323 10.57 MB/sec execute 411 sec latency 63.829 ms 2024-04-25T04:27:08.278 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 218/1719 objects degraded (12.682%), 26 pgs degraded 2024-04-25T04:27:08.278 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-25T04:27:08.278 INFO:teuthology.orchestra.run.smithi078.stdout: osd.3 (root=default,host=smithi078) is down 2024-04-25T04:27:08.278 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 218/1719 objects degraded (12.682%), 26 pgs degraded 2024-04-25T04:27:08.278 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-04-25T04:27:08.279 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-04-25T04:27:08.280 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-04-25T04:27:08.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234598 10.56 MB/sec execute 412 sec latency 51.927 ms 2024-04-25T04:27:09.244 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:09 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[76998]: 2024-04-25T04:27:09.082+0000 7f209c3037c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-25T04:27:09.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:09 smithi078 ceph-mon[62942]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:09.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:09 smithi078 ceph-mon[62942]: pgmap v174: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 218/1719 objects degraded (12.682%) 2024-04-25T04:27:09.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:09.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:09.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:09 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 218/1719 objects degraded (12.682%), 26 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:09.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:09 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1574704669' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:27:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:09 smithi112 ceph-mon[50007]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:09 smithi112 ceph-mon[50007]: pgmap v174: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.6 MiB/s wr, 464 op/s; 218/1719 objects degraded (12.682%) 2024-04-25T04:27:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:09 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 218/1719 objects degraded (12.682%), 26 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:09 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1574704669' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:27:09.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234911 10.55 MB/sec execute 413 sec latency 61.794 ms 2024-04-25T04:27:10.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:10 smithi078 ceph-mon[62942]: pgmap v175: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.8 MiB/s wr, 595 op/s; 214/1638 objects degraded (13.065%) 2024-04-25T04:27:10.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:10 smithi112 ceph-mon[50007]: pgmap v175: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.8 MiB/s wr, 595 op/s; 214/1638 objects degraded (13.065%) 2024-04-25T04:27:10.752 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:10 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[76998]: 2024-04-25T04:27:10.590+0000 7f209c3037c0 -1 osd.3 109 log_to_monitors true 2024-04-25T04:27:10.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235330 10.54 MB/sec execute 414 sec latency 54.274 ms 2024-04-25T04:27:11.257 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:11 smithi078 ceph-mon[62942]: from='osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-25T04:27:11.257 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:27:11 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[76998]: 2024-04-25T04:27:11.233+0000 7f208d75b700 -1 osd.3 109 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:27:11.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:11 smithi112 ceph-mon[50007]: from='osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-25T04:27:11.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235666 10.53 MB/sec execute 415 sec latency 37.497 ms 2024-04-25T04:27:12.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:12 smithi078 ceph-mon[62942]: from='osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-25T04:27:12.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:12 smithi078 ceph-mon[62942]: osdmap e112: 8 total, 7 up, 8 in 2024-04-25T04:27:12.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:12 smithi078 ceph-mon[62942]: from='osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:27:12.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:12.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:12.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:12 smithi078 ceph-mon[62942]: pgmap v177: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 411 op/s; 213/1605 objects degraded (13.271%) 2024-04-25T04:27:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:12 smithi112 ceph-mon[50007]: from='osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-25T04:27:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:12 smithi112 ceph-mon[50007]: osdmap e112: 8 total, 7 up, 8 in 2024-04-25T04:27:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:12 smithi112 ceph-mon[50007]: from='osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-04-25T04:27:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:12 smithi112 ceph-mon[50007]: pgmap v177: 97 pgs: 15 active+undersized, 26 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 411 op/s; 213/1605 objects degraded (13.271%) 2024-04-25T04:27:12.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235854 10.52 MB/sec execute 416 sec latency 612.724 ms 2024-04-25T04:27:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:13 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:27:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:13 smithi078 ceph-mon[62942]: osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330] boot 2024-04-25T04:27:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:13 smithi078 ceph-mon[62942]: osdmap e113: 8 total, 8 up, 8 in 2024-04-25T04:27:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:13 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:27:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:13 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:13 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:13.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:13 smithi078 ceph-mon[62942]: osdmap e114: 8 total, 8 up, 8 in 2024-04-25T04:27:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:13 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:27:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:13 smithi112 ceph-mon[50007]: osd.3 [v2:172.21.15.78:6826/1790246330,v1:172.21.15.78:6827/1790246330] boot 2024-04-25T04:27:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:13 smithi112 ceph-mon[50007]: osdmap e113: 8 total, 8 up, 8 in 2024-04-25T04:27:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:13 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-25T04:27:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:13 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:13 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:13 smithi112 ceph-mon[50007]: osdmap e114: 8 total, 8 up, 8 in 2024-04-25T04:27:13.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235931 10.50 MB/sec execute 417 sec latency 1612.767 ms 2024-04-25T04:27:14.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:14 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 213/1605 objects degraded (13.271%), 26 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:14.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:14 smithi078 ceph-mon[62942]: pgmap v180: 97 pgs: 1 activating, 1 activating+degraded, 4 remapped+peering, 9 active+undersized, 19 active+undersized+degraded, 63 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 564 op/s; 164/1653 objects degraded (9.921%); 14 B/s, 0 objects/s recovering 2024-04-25T04:27:14.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:14 smithi078 ceph-mon[62942]: osdmap e115: 8 total, 8 up, 8 in 2024-04-25T04:27:14.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:14 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 213/1605 objects degraded (13.271%), 26 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:14.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:14 smithi112 ceph-mon[50007]: pgmap v180: 97 pgs: 1 activating, 1 activating+degraded, 4 remapped+peering, 9 active+undersized, 19 active+undersized+degraded, 63 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 564 op/s; 164/1653 objects degraded (9.921%); 14 B/s, 0 objects/s recovering 2024-04-25T04:27:14.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:14 smithi112 ceph-mon[50007]: osdmap e115: 8 total, 8 up, 8 in 2024-04-25T04:27:14.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236142 10.49 MB/sec execute 418 sec latency 2156.398 ms 2024-04-25T04:27:15.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236653 10.51 MB/sec execute 419 sec latency 31.389 ms 2024-04-25T04:27:16.441 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:16 smithi078 ceph-mon[62942]: osdmap e116: 8 total, 8 up, 8 in 2024-04-25T04:27:16.441 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:16 smithi078 ceph-mon[62942]: pgmap v183: 97 pgs: 1 peering, 1 active+recovering+degraded, 1 activating, 1 activating+degraded, 6 remapped+peering, 7 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 67 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.4 MiB/s wr, 426 op/s; 113/1653 objects degraded (6.836%); 1.0 MiB/s, 6 objects/s recovering 2024-04-25T04:27:16.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:16 smithi112 ceph-mon[50007]: osdmap e116: 8 total, 8 up, 8 in 2024-04-25T04:27:16.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:16 smithi112 ceph-mon[50007]: pgmap v183: 97 pgs: 1 peering, 1 active+recovering+degraded, 1 activating, 1 activating+degraded, 6 remapped+peering, 7 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 67 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.4 MiB/s wr, 426 op/s; 113/1653 objects degraded (6.836%); 1.0 MiB/s, 6 objects/s recovering 2024-04-25T04:27:16.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236914 10.51 MB/sec execute 420 sec latency 625.599 ms 2024-04-25T04:27:17.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:17 smithi078 ceph-mon[62942]: osdmap e117: 8 total, 8 up, 8 in 2024-04-25T04:27:17.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:17 smithi112 ceph-mon[50007]: osdmap e117: 8 total, 8 up, 8 in 2024-04-25T04:27:17.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237174 10.51 MB/sec execute 421 sec latency 1012.323 ms 2024-04-25T04:27:18.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:18 smithi078 ceph-mon[62942]: osdmap e118: 8 total, 8 up, 8 in 2024-04-25T04:27:18.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:18 smithi078 ceph-mon[62942]: pgmap v186: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating, 1 activating+degraded, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 84 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 MiB/s wr, 69 op/s; 69/1650 objects degraded (4.182%); 1.1 MiB/s, 19 objects/s recovering 2024-04-25T04:27:18.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:18 smithi078 ceph-mon[62942]: osdmap e119: 8 total, 8 up, 8 in 2024-04-25T04:27:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:18 smithi112 ceph-mon[50007]: osdmap e118: 8 total, 8 up, 8 in 2024-04-25T04:27:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:18 smithi112 ceph-mon[50007]: pgmap v186: 97 pgs: 2 peering, 1 active+recovering+degraded, 1 activating, 1 activating+degraded, 5 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 84 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 MiB/s wr, 69 op/s; 69/1650 objects degraded (4.182%); 1.1 MiB/s, 19 objects/s recovering 2024-04-25T04:27:18.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:18 smithi112 ceph-mon[50007]: osdmap e119: 8 total, 8 up, 8 in 2024-04-25T04:27:18.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237389 10.50 MB/sec execute 422 sec latency 898.712 ms 2024-04-25T04:27:19.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-25T04:27:19.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:19 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 69/1650 objects degraded (4.182%), 5 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:19.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:19.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:19.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:19.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:19.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:19.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-25T04:27:19.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:19 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 69/1650 objects degraded (4.182%), 5 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:19.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237713 10.49 MB/sec execute 423 sec latency 63.448 ms 2024-04-25T04:27:20.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:20 smithi078 ceph-mon[62942]: pgmap v188: 97 pgs: 2 peering, 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 91 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 328 op/s; 58/1704 objects degraded (3.404%); 1.1 MiB/s, 30 objects/s recovering 2024-04-25T04:27:20.551 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:20 smithi112 ceph-mon[50007]: pgmap v188: 97 pgs: 2 peering, 1 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 91 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.1 MiB/s wr, 328 op/s; 58/1704 objects degraded (3.404%); 1.1 MiB/s, 30 objects/s recovering 2024-04-25T04:27:20.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238090 10.48 MB/sec execute 424 sec latency 35.871 ms 2024-04-25T04:27:21.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238402 10.47 MB/sec execute 425 sec latency 38.854 ms 2024-04-25T04:27:22.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:22 smithi078 ceph-mon[62942]: pgmap v189: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.5 MiB/s wr, 525 op/s; 576 KiB/s, 41 objects/s recovering 2024-04-25T04:27:22.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:22 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/1704 objects degraded (3.404%), 3 pgs degraded) 2024-04-25T04:27:22.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:22 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:27:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:22 smithi112 ceph-mon[50007]: pgmap v189: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.5 MiB/s wr, 525 op/s; 576 KiB/s, 41 objects/s recovering 2024-04-25T04:27:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:22 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 58/1704 objects degraded (3.404%), 3 pgs degraded) 2024-04-25T04:27:22.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:22 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:27:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238784 10.48 MB/sec execute 426 sec latency 49.743 ms 2024-04-25T04:27:23.857 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239243 10.48 MB/sec execute 427 sec latency 41.288 ms 2024-04-25T04:27:24.290 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:24.290 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:24.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:24.858 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239635 10.47 MB/sec execute 428 sec latency 33.120 ms 2024-04-25T04:27:25.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:25 smithi078 ceph-mon[62942]: pgmap v190: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.9 MiB/s wr, 446 op/s; 415 KiB/s, 24 objects/s recovering 2024-04-25T04:27:25.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:25 smithi112 ceph-mon[50007]: pgmap v190: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 4.9 MiB/s wr, 446 op/s; 415 KiB/s, 24 objects/s recovering 2024-04-25T04:27:25.858 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240038 10.48 MB/sec execute 429 sec latency 30.768 ms 2024-04-25T04:27:26.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:26 smithi078 ceph-mon[62942]: pgmap v191: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.4 MiB/s wr, 410 op/s; 369 KiB/s, 21 objects/s recovering 2024-04-25T04:27:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:26 smithi112 ceph-mon[50007]: pgmap v191: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.4 MiB/s wr, 410 op/s; 369 KiB/s, 21 objects/s recovering 2024-04-25T04:27:26.858 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240466 10.49 MB/sec execute 430 sec latency 36.146 ms 2024-04-25T04:27:27.859 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240830 10.50 MB/sec execute 431 sec latency 53.599 ms 2024-04-25T04:27:28.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:28 smithi078 ceph-mon[62942]: pgmap v192: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 576 op/s; 329 KiB/s, 19 objects/s recovering 2024-04-25T04:27:28.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:28 smithi112 ceph-mon[50007]: pgmap v192: 97 pgs: 97 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 576 op/s; 329 KiB/s, 19 objects/s recovering 2024-04-25T04:27:28.865 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241115 10.49 MB/sec execute 432 sec latency 54.171 ms 2024-04-25T04:27:29.412 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:29.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:29.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241413 10.47 MB/sec execute 433 sec latency 63.333 ms 2024-04-25T04:27:30.471 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:30 smithi078 ceph-mon[62942]: pgmap v193: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 539 op/s; 276 KiB/s, 16 objects/s recovering 2024-04-25T04:27:30.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:30 smithi112 ceph-mon[50007]: pgmap v193: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 539 op/s; 276 KiB/s, 16 objects/s recovering 2024-04-25T04:27:30.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241601 10.46 MB/sec execute 434 sec latency 62.988 ms 2024-04-25T04:27:31.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241769 10.44 MB/sec execute 435 sec latency 65.593 ms 2024-04-25T04:27:32.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:32 smithi078 ceph-mon[62942]: pgmap v194: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 190 KiB/s, 8 objects/s recovering 2024-04-25T04:27:32.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:32 smithi112 ceph-mon[50007]: pgmap v194: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 190 KiB/s, 8 objects/s recovering 2024-04-25T04:27:32.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241996 10.43 MB/sec execute 436 sec latency 56.823 ms 2024-04-25T04:27:33.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:33.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242219 10.43 MB/sec execute 437 sec latency 69.727 ms 2024-04-25T04:27:33.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:34.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:34 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:34.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:34 smithi078 ceph-mon[62942]: Upgrade: osd.4 is safe to restart 2024-04-25T04:27:34.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:34 smithi078 ceph-mon[62942]: pgmap v195: 97 pgs: 97 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.7 MiB/s wr, 429 op/s 2024-04-25T04:27:34.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242544 10.41 MB/sec execute 438 sec latency 89.963 ms 2024-04-25T04:27:34.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:34 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-25T04:27:34.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:34 smithi112 ceph-mon[50007]: Upgrade: osd.4 is safe to restart 2024-04-25T04:27:34.901 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:34 smithi112 ceph-mon[50007]: pgmap v195: 97 pgs: 97 active+clean; 327 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.7 MiB/s wr, 429 op/s 2024-04-25T04:27:35.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242999 10.42 MB/sec execute 439 sec latency 44.234 ms 2024-04-25T04:27:36.413 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:36 smithi078 ceph-mon[62942]: Upgrade: Updating osd.4 2024-04-25T04:27:36.413 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:36 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:36.414 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:36 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T04:27:36.414 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:36 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:27:36.414 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:36 smithi078 ceph-mon[62942]: Deploying daemon osd.4 on smithi112 2024-04-25T04:27:36.414 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:36 smithi078 ceph-mon[62942]: pgmap v196: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.5 MiB/s wr, 480 op/s 2024-04-25T04:27:36.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:36 smithi112 ceph-mon[50007]: Upgrade: Updating osd.4 2024-04-25T04:27:36.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:36 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:36.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:36 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-25T04:27:36.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:36 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:27:36.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:36 smithi112 ceph-mon[50007]: Deploying daemon osd.4 on smithi112 2024-04-25T04:27:36.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:36 smithi112 ceph-mon[50007]: pgmap v196: 97 pgs: 97 active+clean; 321 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.5 MiB/s wr, 480 op/s 2024-04-25T04:27:36.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243297 10.41 MB/sec execute 440 sec latency 30.751 ms 2024-04-25T04:27:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243635 10.42 MB/sec execute 441 sec latency 54.698 ms 2024-04-25T04:27:38.453 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:38 smithi078 ceph-mon[62942]: pgmap v197: 97 pgs: 97 active+clean; 322 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-04-25T04:27:38.453 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:38.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:38 smithi112 ceph-mon[50007]: pgmap v197: 97 pgs: 97 active+clean; 322 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 542 op/s 2024-04-25T04:27:38.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:38.762 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:27:38.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244136 10.43 MB/sec execute 442 sec latency 87.769 ms 2024-04-25T04:27:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:39 smithi112 ceph-mon[50007]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:39.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:39 smithi078 ceph-mon[62942]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:39.704 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:27:39.704 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 28s ago 16m 24.6M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:27:39.704 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (3m) 28s ago 16m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:27:39.704 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (3m) 3m ago 15m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 28s ago 15m 55.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (9m) 28s ago 9m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (9m) 3m ago 9m 14.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (6m) 28s ago 17m 510M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (5m) 3m ago 15m 450M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (5m) 28s ago 17m 49.9M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (4m) 3m ago 15m 35.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (9m) 28s ago 9m 279M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (9m) 3m ago 9m 85.3M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 28s ago 15m 17.5M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (6m) 3m ago 15m 16.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:27:39.705 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 28s ago 14m 358M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 28s ago 13m 286M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (88s) 28s ago 13m 190M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (38s) 28s ago 12m 13.7M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (12m) 3m ago 12m 528M 2704M 17.2.0 e1d6a67b021e b60ba2d4a62d 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (12m) 3m ago 12m 381M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (11m) 3m ago 11m 556M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (11m) 3m ago 11m 466M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:27:39.706 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 28s ago 14m 73.3M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:27:39.766 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:39 smithi112 systemd[1]: Stopping Ceph osd.4 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:27:39.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244492 10.44 MB/sec execute 443 sec latency 44.384 ms 2024-04-25T04:27:40.151 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[28383]: 2024-04-25T04:27:39.759+0000 7f4c21f92700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:27:40.151 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[28383]: 2024-04-25T04:27:39.759+0000 7f4c21f92700 -1 osd.4 119 *** Got signal Terminated *** 2024-04-25T04:27:40.151 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[28383]: 2024-04-25T04:27:39.759+0000 7f4c21f92700 -1 osd.4 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:27:40.264 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 4 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:27:40.265 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:27:40.266 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:27:40.266 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:27:40.266 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-25T04:27:40.266 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:27:40.266 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:27:40.266 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:27:40.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:40 smithi078 ceph-mon[62942]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:40.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:40 smithi078 ceph-mon[62942]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:40.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:40 smithi078 ceph-mon[62942]: osd.4 marked itself down and dead 2024-04-25T04:27:40.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:40 smithi078 ceph-mon[62942]: pgmap v198: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 468 op/s 2024-04-25T04:27:40.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:40 smithi112 ceph-mon[50007]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:40.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:40 smithi112 ceph-mon[50007]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:40.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:40 smithi112 ceph-mon[50007]: osd.4 marked itself down and dead 2024-04-25T04:27:40.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:40 smithi112 ceph-mon[50007]: pgmap v198: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 468 op/s 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "10/23 daemons upgraded", 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:27:40.664 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:27:40.665 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:27:40.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244698 10.42 MB/sec execute 444 sec latency 551.334 ms 2024-04-25T04:27:41.084 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down 2024-04-25T04:27:41.084 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-25T04:27:41.084 INFO:teuthology.orchestra.run.smithi078.stdout: osd.4 (root=default,host=smithi112) is down 2024-04-25T04:27:41.151 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:40 smithi112 podman[54491]: 2024-04-25 04:27:40.655434731 +0000 UTC m=+1.020113489 container died b60ba2d4a62d2878954c1e031339317f164e9066b7fb0c4dc48b85894fb9e750 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, release=754, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vcs-type=git, maintainer=Guillaume Abrioux ) 2024-04-25T04:27:41.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:41 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:27:41.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:41 smithi078 ceph-mon[62942]: osdmap e120: 8 total, 7 up, 8 in 2024-04-25T04:27:41.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:41 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3371354332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:41.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:41 smithi078 ceph-mon[62942]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:41.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:41 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3348273176' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:27:41.499 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:41 smithi112 podman[54491]: 2024-04-25 04:27:41.242625367 +0000 UTC m=+1.607304125 container cleanup b60ba2d4a62d2878954c1e031339317f164e9066b7fb0c4dc48b85894fb9e750 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vcs-type=git, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, version=8, distribution-scope=public, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux ) 2024-04-25T04:27:41.499 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:41 smithi112 bash[54491]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4 2024-04-25T04:27:41.500 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:41 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:27:41.500 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:41 smithi112 ceph-mon[50007]: osdmap e120: 8 total, 7 up, 8 in 2024-04-25T04:27:41.500 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:41 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3371354332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:41.500 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:41 smithi112 ceph-mon[50007]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:27:41.500 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:41 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3348273176' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:27:41.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244872 10.41 MB/sec execute 445 sec latency 975.157 ms 2024-04-25T04:27:41.901 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:41 smithi112 podman[54513]: 2024-04-25 04:27:41.494085027 +0000 UTC m=+0.834432888 container remove b60ba2d4a62d2878954c1e031339317f164e9066b7fb0c4dc48b85894fb9e750 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, ceph=True, vcs-type=git, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-25T04:27:42.211 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:41 smithi112 podman[54604]: 2024-04-25 04:27:41.928512866 +0000 UTC m=+0.114142003 container create be8b1863377ba8c6122575b20b040af1682b8d2a21d97232b4a87c4b315c9ae7 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-25T04:27:42.211 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:41 smithi112 podman[54604]: 2024-04-25 04:27:41.8354479 +0000 UTC m=+0.021077060 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:27:42.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:42 smithi078 ceph-mon[62942]: osdmap e121: 8 total, 7 up, 8 in 2024-04-25T04:27:42.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:42 smithi078 ceph-mon[62942]: pgmap v201: 97 pgs: 3 active+undersized, 4 active+undersized+degraded, 7 peering, 7 stale+active+clean, 76 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.8 MiB/s wr, 493 op/s; 35/1680 objects degraded (2.083%) 2024-04-25T04:27:42.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:42 smithi112 ceph-mon[50007]: osdmap e121: 8 total, 7 up, 8 in 2024-04-25T04:27:42.575 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:42 smithi112 ceph-mon[50007]: pgmap v201: 97 pgs: 3 active+undersized, 4 active+undersized+degraded, 7 peering, 7 stale+active+clean, 76 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.8 MiB/s wr, 493 op/s; 35/1680 objects degraded (2.083%) 2024-04-25T04:27:42.575 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:42 smithi112 podman[54604]: 2024-04-25 04:27:42.206395534 +0000 UTC m=+0.392024671 container init be8b1863377ba8c6122575b20b040af1682b8d2a21d97232b4a87c4b315c9ae7 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-25T04:27:42.575 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:42 smithi112 podman[54604]: 2024-04-25 04:27:42.210590857 +0000 UTC m=+0.396220003 container start be8b1863377ba8c6122575b20b040af1682b8d2a21d97232b4a87c4b315c9ae7 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-25T04:27:42.575 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:42 smithi112 podman[54604]: 2024-04-25 04:27:42.306422063 +0000 UTC m=+0.492051212 container attach be8b1863377ba8c6122575b20b040af1682b8d2a21d97232b4a87c4b315c9ae7 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-25T04:27:42.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245195 10.39 MB/sec execute 446 sec latency 47.370 ms 2024-04-25T04:27:42.901 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:42 smithi112 podman[54604]: 2024-04-25 04:27:42.569790037 +0000 UTC m=+0.755419192 container died be8b1863377ba8c6122575b20b040af1682b8d2a21d97232b4a87c4b315c9ae7 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-25T04:27:43.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:43 smithi078 ceph-mon[62942]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-25T04:27:43.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:43 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 35/1680 objects degraded (2.083%), 4 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:43.505 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:43 smithi112 ceph-mon[50007]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-25T04:27:43.505 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:43 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 35/1680 objects degraded (2.083%), 4 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:43.506 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:43 smithi112 podman[54630]: 2024-04-25 04:27:43.244558866 +0000 UTC m=+0.668763587 container remove be8b1863377ba8c6122575b20b040af1682b8d2a21d97232b4a87c4b315c9ae7 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-25T04:27:43.506 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:43 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-25T04:27:43.506 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:43 smithi112 systemd[1]: Stopped Ceph osd.4 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:27:43.506 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:43 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.4.service: Consumed 4min 30.091s CPU time. 2024-04-25T04:27:43.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245475 10.38 MB/sec execute 447 sec latency 53.015 ms 2024-04-25T04:27:43.901 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:43 smithi112 systemd[1]: Starting Ceph osd.4 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:27:43.901 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:43 smithi112 podman[54733]: 2024-04-25 04:27:43.628553602 +0000 UTC m=+0.019498069 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:27:43.901 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:43 smithi112 podman[54733]: 2024-04-25 04:27:43.745814583 +0000 UTC m=+0.136759025 container create 9ce857b331e2abd00591da833a10635bf94a70f304d3065edafd524c69a7912f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-25T04:27:44.286 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 podman[54733]: 2024-04-25 04:27:44.165481214 +0000 UTC m=+0.556425664 container init 9ce857b331e2abd00591da833a10635bf94a70f304d3065edafd524c69a7912f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:27:44.287 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 podman[54733]: 2024-04-25 04:27:44.169242572 +0000 UTC m=+0.560187021 container start 9ce857b331e2abd00591da833a10635bf94a70f304d3065edafd524c69a7912f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:27:44.412 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:44 smithi078 ceph-mon[62942]: pgmap v202: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 7 peering, 1 stale+active+clean, 59 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 460 op/s; 208/1695 objects degraded (12.271%) 2024-04-25T04:27:44.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-mon[50007]: pgmap v202: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 7 peering, 1 stale+active+clean, 59 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 460 op/s; 208/1695 objects degraded (12.271%) 2024-04-25T04:27:44.651 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 podman[54733]: 2024-04-25 04:27:44.282507161 +0000 UTC m=+0.673451618 container attach 9ce857b331e2abd00591da833a10635bf94a70f304d3065edafd524c69a7912f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-25T04:27:44.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245853 10.39 MB/sec execute 448 sec latency 59.660 ms 2024-04-25T04:27:45.125 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:27:45.126 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:27:45.126 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:27:45.126 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:27:45.126 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-25T04:27:45.126 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-25T04:27:45.126 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 bash[54733]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate[54747]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-25T04:27:45.127 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:44 smithi112 podman[54733]: 2024-04-25 04:27:44.743659991 +0000 UTC m=+1.134604440 container died 9ce857b331e2abd00591da833a10635bf94a70f304d3065edafd524c69a7912f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-25T04:27:45.799 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:45 smithi112 podman[54870]: 2024-04-25 04:27:45.437087295 +0000 UTC m=+0.688482840 container remove 9ce857b331e2abd00591da833a10635bf94a70f304d3065edafd524c69a7912f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-25T04:27:45.799 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:45 smithi112 podman[54935]: 2024-04-25 04:27:45.565385408 +0000 UTC m=+0.020650233 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:27:45.799 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:45 smithi112 podman[54935]: 2024-04-25 04:27:45.671511347 +0000 UTC m=+0.126776193 container create 77af6e06ee8bc006cb4eec855b4850cda78e08eb7766900b1426e473c97c6b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-25T04:27:45.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246246 10.38 MB/sec execute 449 sec latency 50.952 ms 2024-04-25T04:27:46.454 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:46 smithi078 ceph-mon[62942]: pgmap v203: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 532 op/s; 257/1725 objects degraded (14.899%) 2024-04-25T04:27:46.454 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:46 smithi078 ceph-mon[62942]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-25T04:27:46.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:46 smithi112 ceph-mon[50007]: pgmap v203: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 532 op/s; 257/1725 objects degraded (14.899%) 2024-04-25T04:27:46.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:46 smithi112 ceph-mon[50007]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-25T04:27:46.651 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:46 smithi112 podman[54935]: 2024-04-25 04:27:46.17472912 +0000 UTC m=+0.629993934 container init 77af6e06ee8bc006cb4eec855b4850cda78e08eb7766900b1426e473c97c6b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:27:46.651 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:46 smithi112 podman[54935]: 2024-04-25 04:27:46.178166531 +0000 UTC m=+0.633431345 container start 77af6e06ee8bc006cb4eec855b4850cda78e08eb7766900b1426e473c97c6b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:27:46.651 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:46 smithi112 ceph-osd[54953]: -- 172.21.15.112:0/2332275295 <== mon.1 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558140bb1680 con 0x558140b7b000 2024-04-25T04:27:46.651 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:46 smithi112 bash[54935]: 77af6e06ee8bc006cb4eec855b4850cda78e08eb7766900b1426e473c97c6b8f 2024-04-25T04:27:46.651 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:46 smithi112 systemd[1]: Started Ceph osd.4 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:27:46.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246588 10.38 MB/sec execute 450 sec latency 64.548 ms 2024-04-25T04:27:47.151 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:46 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:27:46.716+0000 7f1858a5d7c0 -1 Falling back to public interface 2024-04-25T04:27:47.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:47 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:47.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:47 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:47.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:47 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:47.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:47 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:47.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:47 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:47.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:47 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:47.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246882 10.37 MB/sec execute 451 sec latency 48.821 ms 2024-04-25T04:27:48.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:48 smithi112 ceph-mon[50007]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 276/1752 objects degraded (15.753%) 2024-04-25T04:27:48.406 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:48 smithi078 ceph-mon[62942]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 276/1752 objects degraded (15.753%) 2024-04-25T04:27:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247238 10.37 MB/sec execute 452 sec latency 58.057 ms 2024-04-25T04:27:49.474 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:49 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 276/1752 objects degraded (15.753%), 28 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:49 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 276/1752 objects degraded (15.753%), 28 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:49.866 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247570 10.38 MB/sec execute 453 sec latency 52.442 ms 2024-04-25T04:27:50.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:50 smithi112 ceph-mon[50007]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 424 op/s; 257/1656 objects degraded (15.519%) 2024-04-25T04:27:50.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:50 smithi078 ceph-mon[62942]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 424 op/s; 257/1656 objects degraded (15.519%) 2024-04-25T04:27:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247827 10.38 MB/sec execute 454 sec latency 56.317 ms 2024-04-25T04:27:51.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:51.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:51.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248021 10.37 MB/sec execute 455 sec latency 66.739 ms 2024-04-25T04:27:52.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:52.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:52.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:52 smithi078 ceph-mon[62942]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 480 op/s; 266/1683 objects degraded (15.805%) 2024-04-25T04:27:52.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:52 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:52.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:52 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:52.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248246 10.36 MB/sec execute 456 sec latency 97.953 ms 2024-04-25T04:27:53.042 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:52 smithi112 ceph-mon[50007]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 480 op/s; 266/1683 objects degraded (15.805%) 2024-04-25T04:27:53.043 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:52 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:53.043 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:52 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:53.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248522 10.34 MB/sec execute 457 sec latency 47.320 ms 2024-04-25T04:27:54.046 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:53 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:27:53.706+0000 7f1858a5d7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-25T04:27:54.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:54.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:54.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:54 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 266/1683 objects degraded (15.805%), 28 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:54.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:54.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:27:54.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:54 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 266/1683 objects degraded (15.805%), 28 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:54.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248747 10.33 MB/sec execute 458 sec latency 47.717 ms 2024-04-25T04:27:55.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:55 smithi078 ceph-mon[62942]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.1 MiB/s wr, 522 op/s; 273/1686 objects degraded (16.192%) 2024-04-25T04:27:55.562 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:55 smithi112 ceph-mon[50007]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.1 MiB/s wr, 522 op/s; 273/1686 objects degraded (16.192%) 2024-04-25T04:27:55.562 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:55 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:27:55.236+0000 7f1858a5d7c0 -1 osd.4 119 log_to_monitors true 2024-04-25T04:27:55.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249042 10.33 MB/sec execute 459 sec latency 60.462 ms 2024-04-25T04:27:56.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:56 smithi078 ceph-mon[62942]: from='osd.4 [v2:172.21.15.112:6800/2303028788,v1:172.21.15.112:6801/2303028788]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:27:56.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:56 smithi078 ceph-mon[62942]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:27:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:56 smithi112 ceph-mon[50007]: from='osd.4 [v2:172.21.15.112:6800/2303028788,v1:172.21.15.112:6801/2303028788]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:27:56.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:56 smithi112 ceph-mon[50007]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-25T04:27:56.651 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:27:56 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:27:56.219+0000 7f1849eb5700 -1 osd.4 119 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:27:56.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249344 10.32 MB/sec execute 460 sec latency 61.012 ms 2024-04-25T04:27:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 462 op/s; 276/1707 objects degraded (16.169%) 2024-04-25T04:27:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-25T04:27:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='osd.4 [v2:172.21.15.112:6800/2303028788,v1:172.21.15.112:6801/2303028788]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:27:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: osdmap e122: 8 total, 7 up, 8 in 2024-04-25T04:27:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:27:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:57.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.4 MiB/s wr, 462 op/s; 276/1707 objects degraded (16.169%) 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='osd.4 [v2:172.21.15.112:6800/2303028788,v1:172.21.15.112:6801/2303028788]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: osdmap e122: 8 total, 7 up, 8 in 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:27:57.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:27:57.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:57.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:57.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:27:57.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:27:57.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249496 10.31 MB/sec execute 461 sec latency 622.874 ms 2024-04-25T04:27:58.448 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:58 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:27:58.448 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:58 smithi078 ceph-mon[62942]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-25T04:27:58.448 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:58 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:27:58.448 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:58 smithi078 ceph-mon[62942]: osd.4 [v2:172.21.15.112:6800/2303028788,v1:172.21.15.112:6801/2303028788] boot 2024-04-25T04:27:58.448 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:58 smithi078 ceph-mon[62942]: osdmap e123: 8 total, 8 up, 8 in 2024-04-25T04:27:58.448 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:58 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:27:58.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:58 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:27:58.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:58 smithi112 ceph-mon[50007]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-04-25T04:27:58.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:58 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:27:58.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:58 smithi112 ceph-mon[50007]: osd.4 [v2:172.21.15.112:6800/2303028788,v1:172.21.15.112:6801/2303028788] boot 2024-04-25T04:27:58.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:58 smithi112 ceph-mon[50007]: osdmap e123: 8 total, 8 up, 8 in 2024-04-25T04:27:58.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:58 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-25T04:27:58.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249607 10.29 MB/sec execute 462 sec latency 1591.199 ms 2024-04-25T04:27:59.460 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:59 smithi078 ceph-mon[62942]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.8 MiB/s wr, 385 op/s; 276/1698 objects degraded (16.254%) 2024-04-25T04:27:59.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:59 smithi078 ceph-mon[62942]: osdmap e124: 8 total, 8 up, 8 in 2024-04-25T04:27:59.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:27:59 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 276/1698 objects degraded (16.254%), 28 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:59.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:59 smithi112 ceph-mon[50007]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.8 MiB/s wr, 385 op/s; 276/1698 objects degraded (16.254%) 2024-04-25T04:27:59.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:59 smithi112 ceph-mon[50007]: osdmap e124: 8 total, 8 up, 8 in 2024-04-25T04:27:59.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:27:59 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 276/1698 objects degraded (16.254%), 28 pgs degraded (PG_DEGRADED) 2024-04-25T04:27:59.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249801 10.28 MB/sec execute 463 sec latency 2027.811 ms 2024-04-25T04:28:00.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:00 smithi078 ceph-mon[62942]: osdmap e125: 8 total, 8 up, 8 in 2024-04-25T04:28:00.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:00 smithi078 ceph-mon[62942]: pgmap v214: 97 pgs: 2 active+recovery_wait+degraded, 3 activating+degraded, 1 activating, 7 remapped+peering, 12 active+undersized, 1 active+recovering, 9 active+undersized+degraded, 62 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 3.3 MiB/s wr, 396 op/s; 181/1629 objects degraded (11.111%); 0 B/s, 6 objects/s recovering 2024-04-25T04:28:00.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:00 smithi112 ceph-mon[50007]: osdmap e125: 8 total, 8 up, 8 in 2024-04-25T04:28:00.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:00 smithi112 ceph-mon[50007]: pgmap v214: 97 pgs: 2 active+recovery_wait+degraded, 3 activating+degraded, 1 activating, 7 remapped+peering, 12 active+undersized, 1 active+recovering, 9 active+undersized+degraded, 62 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 3.3 MiB/s wr, 396 op/s; 181/1629 objects degraded (11.111%); 0 B/s, 6 objects/s recovering 2024-04-25T04:28:00.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250154 10.27 MB/sec execute 464 sec latency 80.907 ms 2024-04-25T04:28:01.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:01 smithi078 ceph-mon[62942]: osdmap e126: 8 total, 8 up, 8 in 2024-04-25T04:28:01.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:01 smithi112 ceph-mon[50007]: osdmap e126: 8 total, 8 up, 8 in 2024-04-25T04:28:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250422 10.26 MB/sec execute 465 sec latency 44.470 ms 2024-04-25T04:28:02.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:02 smithi078 ceph-mon[62942]: osdmap e127: 8 total, 8 up, 8 in 2024-04-25T04:28:02.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:02 smithi078 ceph-mon[62942]: pgmap v217: 97 pgs: 5 active+recovery_wait+degraded, 3 activating+degraded, 1 activating, 7 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 79 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 418 op/s; 128/1548 objects degraded (8.269%); 289 KiB/s, 53 keys/s, 23 objects/s recovering 2024-04-25T04:28:02.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:02 smithi112 ceph-mon[50007]: osdmap e127: 8 total, 8 up, 8 in 2024-04-25T04:28:02.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:02 smithi112 ceph-mon[50007]: pgmap v217: 97 pgs: 5 active+recovery_wait+degraded, 3 activating+degraded, 1 activating, 7 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 79 active+clean; 324 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 418 op/s; 128/1548 objects degraded (8.269%); 289 KiB/s, 53 keys/s, 23 objects/s recovering 2024-04-25T04:28:02.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250653 10.25 MB/sec execute 466 sec latency 637.780 ms 2024-04-25T04:28:03.287 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:03 smithi078 ceph-mon[62942]: osdmap e128: 8 total, 8 up, 8 in 2024-04-25T04:28:03.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:03 smithi112 ceph-mon[50007]: osdmap e128: 8 total, 8 up, 8 in 2024-04-25T04:28:03.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250988 10.27 MB/sec execute 467 sec latency 1025.664 ms 2024-04-25T04:28:04.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:04 smithi078 ceph-mon[62942]: osdmap e129: 8 total, 8 up, 8 in 2024-04-25T04:28:04.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:04 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 128/1548 objects degraded (8.269%), 9 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:04.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:04 smithi078 ceph-mon[62942]: pgmap v220: 97 pgs: 1 peering, 3 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 90 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 315 op/s; 48/1569 objects degraded (3.059%); 2.5 MiB/s, 53 keys/s, 58 objects/s recovering 2024-04-25T04:28:04.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:04 smithi112 ceph-mon[50007]: osdmap e129: 8 total, 8 up, 8 in 2024-04-25T04:28:04.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:04 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 128/1548 objects degraded (8.269%), 9 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:04.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:04 smithi112 ceph-mon[50007]: pgmap v220: 97 pgs: 1 peering, 3 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 90 active+clean; 323 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 315 op/s; 48/1569 objects degraded (3.059%); 2.5 MiB/s, 53 keys/s, 58 objects/s recovering 2024-04-25T04:28:04.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251279 10.26 MB/sec execute 468 sec latency 1024.033 ms 2024-04-25T04:28:05.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:05 smithi078 ceph-mon[62942]: osdmap e130: 8 total, 8 up, 8 in 2024-04-25T04:28:05.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:05 smithi112 ceph-mon[50007]: osdmap e130: 8 total, 8 up, 8 in 2024-04-25T04:28:05.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251572 10.26 MB/sec execute 469 sec latency 84.906 ms 2024-04-25T04:28:06.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:06 smithi078 ceph-mon[62942]: pgmap v222: 97 pgs: 1 peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 451 op/s; 48/1605 objects degraded (2.991%); 1.8 MiB/s, 43 keys/s, 46 objects/s recovering 2024-04-25T04:28:06.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:06 smithi112 ceph-mon[50007]: pgmap v222: 97 pgs: 1 peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 451 op/s; 48/1605 objects degraded (2.991%); 1.8 MiB/s, 43 keys/s, 46 objects/s recovering 2024-04-25T04:28:06.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251771 10.26 MB/sec execute 470 sec latency 55.143 ms 2024-04-25T04:28:07.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251940 10.24 MB/sec execute 471 sec latency 95.221 ms 2024-04-25T04:28:08.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252212 10.22 MB/sec execute 472 sec latency 65.617 ms 2024-04-25T04:28:09.240 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:09 smithi078 ceph-mon[62942]: pgmap v223: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 1.5 MiB/s, 40 objects/s recovering 2024-04-25T04:28:09.241 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:09.241 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:09.241 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:09 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/1605 objects degraded (2.991%), 4 pgs degraded) 2024-04-25T04:28:09.241 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:09 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:28:09.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:09 smithi112 ceph-mon[50007]: pgmap v223: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 1.5 MiB/s, 40 objects/s recovering 2024-04-25T04:28:09.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:09 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/1605 objects degraded (2.991%), 4 pgs degraded) 2024-04-25T04:28:09.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:09 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:28:09.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252511 10.21 MB/sec execute 473 sec latency 63.571 ms 2024-04-25T04:28:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252956 10.21 MB/sec execute 474 sec latency 60.401 ms 2024-04-25T04:28:11.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:11 smithi078 ceph-mon[62942]: pgmap v224: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 340 op/s; 1.1 MiB/s, 30 objects/s recovering 2024-04-25T04:28:11.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:11 smithi112 ceph-mon[50007]: pgmap v224: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 340 op/s; 1.1 MiB/s, 30 objects/s recovering 2024-04-25T04:28:11.704 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:28:11.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253382 10.22 MB/sec execute 475 sec latency 51.016 ms 2024-04-25T04:28:12.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:28:12.461 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:28:12.724 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:28:12.724 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 61s ago 16m 24.6M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:28:12.724 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 61s ago 16m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (4m) 21s ago 15m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 61s ago 16m 55.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (10m) 61s ago 10m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (10m) 21s ago 10m 15.2M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (6m) 61s ago 17m 510M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (6m) 21s ago 15m 452M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (5m) 61s ago 18m 49.9M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (5m) 21s ago 15m 40.5M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (10m) 61s ago 10m 279M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (10m) 21s ago 10m 87.9M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 61s ago 16m 17.5M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 21s ago 15m 19.8M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 61s ago 14m 358M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 61s ago 14m 286M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:28:12.725 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 61s ago 13m 190M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:28:12.726 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (71s) 61s ago 13m 13.7M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:28:12.726 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (27s) 21s ago 13m 12.7M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:28:12.726 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (12m) 21s ago 12m 593M 2704M 17.2.0 e1d6a67b021e 1ec21f84809d 2024-04-25T04:28:12.726 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (12m) 21s ago 12m 512M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:28:12.726 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (11m) 21s ago 11m 567M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:28:12.726 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 61s ago 15m 73.3M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:28:12.870 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253906 10.22 MB/sec execute 476 sec latency 35.064 ms 2024-04-25T04:28:13.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:13 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:28:13.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:13 smithi078 ceph-mon[62942]: Upgrade: osd.5 is safe to restart 2024-04-25T04:28:13.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:13 smithi078 ceph-mon[62942]: from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:13.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:13 smithi078 ceph-mon[62942]: pgmap v225: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 447 op/s; 1.0 MiB/s, 27 objects/s recovering 2024-04-25T04:28:13.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:13 smithi078 ceph-mon[62942]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:13.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:13 smithi078 ceph-mon[62942]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 5 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:13.361 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:28:13.362 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:28:13.362 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:13.362 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:28:13.362 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-25T04:28:13.362 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 9 2024-04-25T04:28:13.362 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:28:13.362 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:28:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:13 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-25T04:28:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:13 smithi112 ceph-mon[50007]: Upgrade: osd.5 is safe to restart 2024-04-25T04:28:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:13 smithi112 ceph-mon[50007]: from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:13 smithi112 ceph-mon[50007]: pgmap v225: 97 pgs: 97 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 447 op/s; 1.0 MiB/s, 27 objects/s recovering 2024-04-25T04:28:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:13 smithi112 ceph-mon[50007]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:13.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:13 smithi112 ceph-mon[50007]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:13.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254135 10.22 MB/sec execute 477 sec latency 37.233 ms 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "11/23 daemons upgraded", 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:28:13.969 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/332203797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: Upgrade: Updating osd.5 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: Deploying daemon osd.5 on smithi112 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: from='client.44309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:14.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:14 smithi078 ceph-mon[62942]: pgmap v226: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 472 op/s; 18 KiB/s, 7 objects/s recovering 2024-04-25T04:28:14.348 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/332203797' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:14.349 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: Upgrade: Updating osd.5 2024-04-25T04:28:14.349 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:14.349 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-25T04:28:14.349 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:28:14.349 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: Deploying daemon osd.5 on smithi112 2024-04-25T04:28:14.349 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: from='client.44309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:14.349 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:14 smithi112 ceph-mon[50007]: pgmap v226: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 472 op/s; 18 KiB/s, 7 objects/s recovering 2024-04-25T04:28:14.594 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:28:14.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254486 10.23 MB/sec execute 478 sec latency 59.892 ms 2024-04-25T04:28:15.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:15 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2719653103' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:28:15.840 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:15 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2719653103' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:28:15.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254816 10.22 MB/sec execute 479 sec latency 33.603 ms 2024-04-25T04:28:16.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:16 smithi078 ceph-mon[62942]: pgmap v227: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 401 op/s; 15 KiB/s, 4 objects/s recovering 2024-04-25T04:28:16.765 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:16 smithi112 ceph-mon[50007]: pgmap v227: 97 pgs: 97 active+clean; 330 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 401 op/s; 15 KiB/s, 4 objects/s recovering 2024-04-25T04:28:16.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255144 10.23 MB/sec execute 480 sec latency 58.236 ms 2024-04-25T04:28:17.151 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:16 smithi112 systemd[1]: Stopping Ceph osd.5 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:28:17.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:17 smithi078 ceph-mon[62942]: osd.5 marked itself down and dead 2024-04-25T04:28:17.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:17 smithi112 ceph-mon[50007]: osd.5 marked itself down and dead 2024-04-25T04:28:17.651 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:17 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[31360]: 2024-04-25T04:28:17.256+0000 7f6f242d1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:28:17.651 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:17 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[31360]: 2024-04-25T04:28:17.256+0000 7f6f242d1700 -1 osd.5 130 *** Got signal Terminated *** 2024-04-25T04:28:17.651 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:17 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[31360]: 2024-04-25T04:28:17.256+0000 7f6f242d1700 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:28:17.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255344 10.22 MB/sec execute 481 sec latency 350.969 ms 2024-04-25T04:28:18.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:18 smithi112 podman[56753]: 2024-04-25 04:28:18.134511855 +0000 UTC m=+1.007844631 container died 1ec21f84809d58e80ecaf5ea459cf5df149bc3e68d8518cac2f0fa77b50a75d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., release=754, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, ceph=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-25T04:28:18.640 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:18 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:28:18.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:18 smithi078 ceph-mon[62942]: osdmap e131: 8 total, 7 up, 8 in 2024-04-25T04:28:18.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:18 smithi078 ceph-mon[62942]: pgmap v229: 97 pgs: 12 stale+active+clean, 85 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 500 op/s; 0 B/s, 0 objects/s recovering 2024-04-25T04:28:18.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:18 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:18.806 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:18 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:28:18.806 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:18 smithi112 ceph-mon[50007]: osdmap e131: 8 total, 7 up, 8 in 2024-04-25T04:28:18.806 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:18 smithi112 ceph-mon[50007]: pgmap v229: 97 pgs: 12 stale+active+clean, 85 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 500 op/s; 0 B/s, 0 objects/s recovering 2024-04-25T04:28:18.807 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:18 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:18.807 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:18 smithi112 podman[56753]: 2024-04-25 04:28:18.559404556 +0000 UTC m=+1.432737339 container cleanup 1ec21f84809d58e80ecaf5ea459cf5df149bc3e68d8518cac2f0fa77b50a75d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, name=centos-stream) 2024-04-25T04:28:18.807 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:18 smithi112 bash[56753]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5 2024-04-25T04:28:18.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255496 10.20 MB/sec execute 482 sec latency 1014.642 ms 2024-04-25T04:28:19.150 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:18 smithi112 podman[56780]: 2024-04-25 04:28:18.801011031 +0000 UTC m=+0.667724352 container remove 1ec21f84809d58e80ecaf5ea459cf5df149bc3e68d8518cac2f0fa77b50a75d6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, vcs-type=git, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-04-25T04:28:19.518 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:19 smithi112 ceph-mon[50007]: osdmap e132: 8 total, 7 up, 8 in 2024-04-25T04:28:19.519 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:19 smithi112 podman[56867]: 2024-04-25 04:28:19.151495353 +0000 UTC m=+0.019830678 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:28:19.519 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:19 smithi112 podman[56867]: 2024-04-25 04:28:19.268912704 +0000 UTC m=+0.137248016 container create c01478a82c9fc7cba2e385f81c88eef7f03fd185bc8f7de568fd1609e98aa27d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:28:19.519 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:19 smithi112 podman[56867]: 2024-04-25 04:28:19.513625546 +0000 UTC m=+0.381960861 container init c01478a82c9fc7cba2e385f81c88eef7f03fd185bc8f7de568fd1609e98aa27d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-25T04:28:19.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:19 smithi078 ceph-mon[62942]: osdmap e132: 8 total, 7 up, 8 in 2024-04-25T04:28:19.858 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:19 smithi112 podman[56867]: 2024-04-25 04:28:19.517217266 +0000 UTC m=+0.385552586 container start c01478a82c9fc7cba2e385f81c88eef7f03fd185bc8f7de568fd1609e98aa27d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-25T04:28:19.858 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:19 smithi112 podman[56867]: 2024-04-25 04:28:19.613651699 +0000 UTC m=+0.481987013 container attach c01478a82c9fc7cba2e385f81c88eef7f03fd185bc8f7de568fd1609e98aa27d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-25T04:28:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255757 10.19 MB/sec execute 483 sec latency 70.265 ms 2024-04-25T04:28:20.150 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:19 smithi112 podman[56867]: 2024-04-25 04:28:19.853296359 +0000 UTC m=+0.721631678 container died c01478a82c9fc7cba2e385f81c88eef7f03fd185bc8f7de568fd1609e98aa27d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-25T04:28:20.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:20 smithi078 ceph-mon[62942]: pgmap v231: 97 pgs: 3 active+undersized, 8 active+undersized+degraded, 3 peering, 8 stale+active+clean, 75 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.9 MiB/s wr, 462 op/s; 56/1656 objects degraded (3.382%) 2024-04-25T04:28:20.715 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:20 smithi112 ceph-mon[50007]: pgmap v231: 97 pgs: 3 active+undersized, 8 active+undersized+degraded, 3 peering, 8 stale+active+clean, 75 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.9 MiB/s wr, 462 op/s; 56/1656 objects degraded (3.382%) 2024-04-25T04:28:20.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256077 10.18 MB/sec execute 484 sec latency 39.481 ms 2024-04-25T04:28:20.969 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:20 smithi112 podman[56893]: 2024-04-25 04:28:20.710898695 +0000 UTC m=+0.848601217 container remove c01478a82c9fc7cba2e385f81c88eef7f03fd185bc8f7de568fd1609e98aa27d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-25T04:28:20.969 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:20 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-25T04:28:20.969 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:20 smithi112 systemd[1]: Stopped Ceph osd.5 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:28:20.969 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:20 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.5.service: Consumed 5min 26.077s CPU time. 2024-04-25T04:28:21.225 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:20 smithi112 systemd[1]: Starting Ceph osd.5 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:28:21.225 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:21 smithi112 podman[56993]: 2024-04-25 04:28:21.092434057 +0000 UTC m=+0.020112161 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:28:21.225 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:21 smithi112 podman[56993]: 2024-04-25 04:28:21.220161418 +0000 UTC m=+0.147839504 container create b039adc7210e4ebc2a8d8558b001f7c808f5951198b47e01c8f62a6116f0edfa (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-25T04:28:21.484 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:21 smithi112 podman[56993]: 2024-04-25 04:28:21.456572874 +0000 UTC m=+0.384250975 container init b039adc7210e4ebc2a8d8558b001f7c808f5951198b47e01c8f62a6116f0edfa (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-25T04:28:21.484 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:21 smithi112 podman[56993]: 2024-04-25 04:28:21.4604365 +0000 UTC m=+0.388114591 container start b039adc7210e4ebc2a8d8558b001f7c808f5951198b47e01c8f62a6116f0edfa (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-04-25T04:28:21.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:21 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 56/1656 objects degraded (3.382%), 8 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:21.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256400 10.17 MB/sec execute 485 sec latency 52.393 ms 2024-04-25T04:28:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:21 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 56/1656 objects degraded (3.382%), 8 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:21.901 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:21 smithi112 podman[56993]: 2024-04-25 04:28:21.556920901 +0000 UTC m=+0.484599003 container attach b039adc7210e4ebc2a8d8558b001f7c808f5951198b47e01c8f62a6116f0edfa (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:28:22.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:28:22.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:28:22.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:28:22.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:28:22.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-25T04:28:22.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-25T04:28:22.401 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 bash[56993]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate[57008]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-25T04:28:22.402 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 podman[56993]: 2024-04-25 04:28:22.03800065 +0000 UTC m=+0.965678745 container died b039adc7210e4ebc2a8d8558b001f7c808f5951198b47e01c8f62a6116f0edfa (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-25T04:28:22.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:22 smithi078 ceph-mon[62942]: pgmap v232: 97 pgs: 9 active+undersized, 18 active+undersized+degraded, 3 peering, 67 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 142/1599 objects degraded (8.881%) 2024-04-25T04:28:22.817 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:22 smithi112 ceph-mon[50007]: pgmap v232: 97 pgs: 9 active+undersized, 18 active+undersized+degraded, 3 peering, 67 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 142/1599 objects degraded (8.881%) 2024-04-25T04:28:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256793 10.17 MB/sec execute 486 sec latency 47.365 ms 2024-04-25T04:28:23.068 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:22 smithi112 podman[57146]: 2024-04-25 04:28:22.812058801 +0000 UTC m=+0.768614829 container remove b039adc7210e4ebc2a8d8558b001f7c808f5951198b47e01c8f62a6116f0edfa (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-activate, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:28:23.068 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 podman[57206]: 2024-04-25 04:28:22.942646035 +0000 UTC m=+0.022870854 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:28:23.068 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 podman[57206]: 2024-04-25 04:28:23.063137036 +0000 UTC m=+0.143361831 container create 789440173ccc330489d30192918e85d63579bfdb2b9e9e0aefd3652fd0167c7d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0) 2024-04-25T04:28:23.650 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 podman[57206]: 2024-04-25 04:28:23.32465839 +0000 UTC m=+0.404883192 container init 789440173ccc330489d30192918e85d63579bfdb2b9e9e0aefd3652fd0167c7d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-25T04:28:23.651 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 podman[57206]: 2024-04-25 04:28:23.327868259 +0000 UTC m=+0.408093054 container start 789440173ccc330489d30192918e85d63579bfdb2b9e9e0aefd3652fd0167c7d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-25T04:28:23.651 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 ceph-osd[57226]: -- 172.21.15.112:0/2868416392 <== mon.1 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c95b169680 con 0x55c95b132c00 2024-04-25T04:28:23.651 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 bash[57206]: 789440173ccc330489d30192918e85d63579bfdb2b9e9e0aefd3652fd0167c7d 2024-04-25T04:28:23.651 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 systemd[1]: Started Ceph osd.5 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:28:23.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257226 10.18 MB/sec execute 487 sec latency 77.992 ms 2024-04-25T04:28:24.150 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:23 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:28:23.871+0000 7f4a058247c0 -1 Falling back to public interface 2024-04-25T04:28:24.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:24.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:24.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:24.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:24.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:28:24.517 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:24.517 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:24.517 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:24.517 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:24.517 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:28:24.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257680 10.19 MB/sec execute 488 sec latency 40.388 ms 2024-04-25T04:28:25.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:25 smithi078 ceph-mon[62942]: pgmap v233: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 598 op/s; 173/1632 objects degraded (10.600%) 2024-04-25T04:28:25.537 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:25 smithi112 ceph-mon[50007]: pgmap v233: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 598 op/s; 173/1632 objects degraded (10.600%) 2024-04-25T04:28:25.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258117 10.20 MB/sec execute 489 sec latency 30.328 ms 2024-04-25T04:28:26.456 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:26 smithi078 ceph-mon[62942]: pgmap v234: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 380 op/s; 173/1629 objects degraded (10.620%) 2024-04-25T04:28:26.456 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:26 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 173/1629 objects degraded (10.620%), 20 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:26.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:26 smithi112 ceph-mon[50007]: pgmap v234: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 328 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 380 op/s; 173/1629 objects degraded (10.620%) 2024-04-25T04:28:26.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:26 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 173/1629 objects degraded (10.620%), 20 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:26.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258659 10.21 MB/sec execute 490 sec latency 34.763 ms 2024-04-25T04:28:27.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259163 10.21 MB/sec execute 491 sec latency 30.218 ms 2024-04-25T04:28:28.760 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:28 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:28.761 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:28 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:28.761 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:28 smithi078 ceph-mon[62942]: pgmap v235: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 389 op/s; 174/1686 objects degraded (10.320%) 2024-04-25T04:28:28.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259516 10.21 MB/sec execute 492 sec latency 54.876 ms 2024-04-25T04:28:29.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:28 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:29.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:28 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:29.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:28 smithi112 ceph-mon[50007]: pgmap v235: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 389 op/s; 174/1686 objects degraded (10.320%) 2024-04-25T04:28:29.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259835 10.20 MB/sec execute 493 sec latency 74.048 ms 2024-04-25T04:28:30.294 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:30.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:30 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:30.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:30.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:30 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:30.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260235 10.20 MB/sec execute 494 sec latency 41.890 ms 2024-04-25T04:28:31.081 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:31 smithi112 ceph-mon[50007]: pgmap v236: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 463 op/s; 173/1785 objects degraded (9.692%) 2024-04-25T04:28:31.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:31 smithi078 ceph-mon[62942]: pgmap v236: 97 pgs: 10 active+undersized, 20 active+undersized+degraded, 67 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 463 op/s; 173/1785 objects degraded (9.692%) 2024-04-25T04:28:31.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260613 10.20 MB/sec execute 495 sec latency 53.710 ms 2024-04-25T04:28:32.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:32 smithi078 ceph-mon[62942]: pgmap v237: 97 pgs: 9 active+undersized, 21 active+undersized+degraded, 67 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.9 MiB/s wr, 632 op/s; 177/1800 objects degraded (9.833%) 2024-04-25T04:28:32.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:32 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 177/1800 objects degraded (9.833%), 21 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:32.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:32 smithi112 ceph-mon[50007]: pgmap v237: 97 pgs: 9 active+undersized, 21 active+undersized+degraded, 67 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.9 MiB/s wr, 632 op/s; 177/1800 objects degraded (9.833%) 2024-04-25T04:28:32.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:32 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 177/1800 objects degraded (9.833%), 21 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:32.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 261064 10.21 MB/sec execute 496 sec latency 43.940 ms 2024-04-25T04:28:33.400 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:28:33.055+0000 7f4a058247c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-25T04:28:33.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 261463 10.21 MB/sec execute 497 sec latency 36.163 ms 2024-04-25T04:28:34.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:34.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:34.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:34 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:34.400 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:28:34.360+0000 7f4a058247c0 -1 osd.5 130 log_to_monitors true 2024-04-25T04:28:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:34.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:34 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:34.867 INFO:teuthology.orchestra.run.smithi078.stdout: 5 261875 10.22 MB/sec execute 498 sec latency 40.302 ms 2024-04-25T04:28:35.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:35 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:35.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:35 smithi078 ceph-mon[62942]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-25T04:28:35.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:35 smithi078 ceph-mon[62942]: pgmap v238: 97 pgs: 9 active+undersized, 21 active+undersized+degraded, 67 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 569 op/s; 175/1686 objects degraded (10.380%) 2024-04-25T04:28:35.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:35 smithi078 ceph-mon[62942]: from='osd.5 [v2:172.21.15.112:6808/1144700700,v1:172.21.15.112:6809/1144700700]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:28:35.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:35 smithi078 ceph-mon[62942]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:28:35.400 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:28:35 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:28:35.157+0000 7f49f6c7c700 -1 osd.5 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:28:35.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:35 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:35.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:35 smithi112 ceph-mon[50007]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-25T04:28:35.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:35 smithi112 ceph-mon[50007]: pgmap v238: 97 pgs: 9 active+undersized, 21 active+undersized+degraded, 67 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 569 op/s; 175/1686 objects degraded (10.380%) 2024-04-25T04:28:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:35 smithi112 ceph-mon[50007]: from='osd.5 [v2:172.21.15.112:6808/1144700700,v1:172.21.15.112:6809/1144700700]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:28:35.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:35 smithi112 ceph-mon[50007]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-25T04:28:35.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262269 10.22 MB/sec execute 499 sec latency 26.663 ms 2024-04-25T04:28:36.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:36 smithi078 ceph-mon[62942]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-25T04:28:36.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:36 smithi078 ceph-mon[62942]: from='osd.5 [v2:172.21.15.112:6808/1144700700,v1:172.21.15.112:6809/1144700700]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:28:36.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:36 smithi078 ceph-mon[62942]: osdmap e133: 8 total, 7 up, 8 in 2024-04-25T04:28:36.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:36 smithi078 ceph-mon[62942]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:28:36.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:36 smithi112 ceph-mon[50007]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-25T04:28:36.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:36 smithi112 ceph-mon[50007]: from='osd.5 [v2:172.21.15.112:6808/1144700700,v1:172.21.15.112:6809/1144700700]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:28:36.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:36 smithi112 ceph-mon[50007]: osdmap e133: 8 total, 7 up, 8 in 2024-04-25T04:28:36.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:36 smithi112 ceph-mon[50007]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:28:36.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262409 10.21 MB/sec execute 500 sec latency 721.839 ms 2024-04-25T04:28:37.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:37 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:28:37.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:37 smithi078 ceph-mon[62942]: osd.5 [v2:172.21.15.112:6808/1144700700,v1:172.21.15.112:6809/1144700700] boot 2024-04-25T04:28:37.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:37 smithi078 ceph-mon[62942]: osdmap e134: 8 total, 8 up, 8 in 2024-04-25T04:28:37.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:37 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:28:37.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:37 smithi078 ceph-mon[62942]: pgmap v241: 97 pgs: 9 active+undersized, 21 active+undersized+degraded, 67 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 602 op/s; 175/1686 objects degraded (10.380%) 2024-04-25T04:28:37.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:37 smithi078 ceph-mon[62942]: osdmap e135: 8 total, 8 up, 8 in 2024-04-25T04:28:37.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:37 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:28:37.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:37 smithi112 ceph-mon[50007]: osd.5 [v2:172.21.15.112:6808/1144700700,v1:172.21.15.112:6809/1144700700] boot 2024-04-25T04:28:37.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:37 smithi112 ceph-mon[50007]: osdmap e134: 8 total, 8 up, 8 in 2024-04-25T04:28:37.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:37 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-25T04:28:37.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:37 smithi112 ceph-mon[50007]: pgmap v241: 97 pgs: 9 active+undersized, 21 active+undersized+degraded, 67 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 602 op/s; 175/1686 objects degraded (10.380%) 2024-04-25T04:28:37.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:37 smithi112 ceph-mon[50007]: osdmap e135: 8 total, 8 up, 8 in 2024-04-25T04:28:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262409 10.19 MB/sec execute 501 sec latency 1721.904 ms 2024-04-25T04:28:38.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262457 10.17 MB/sec execute 502 sec latency 2088.632 ms 2024-04-25T04:28:39.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:39 smithi112 ceph-mon[50007]: osdmap e136: 8 total, 8 up, 8 in 2024-04-25T04:28:39.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:39 smithi112 ceph-mon[50007]: pgmap v244: 97 pgs: 6 peering, 6 active+undersized, 18 active+undersized+degraded, 67 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s wr, 165 op/s; 146/1713 objects degraded (8.523%) 2024-04-25T04:28:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:39 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 175/1686 objects degraded (10.380%), 21 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:39.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:39 smithi112 ceph-mon[50007]: osdmap e137: 8 total, 8 up, 8 in 2024-04-25T04:28:39.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:39 smithi078 ceph-mon[62942]: osdmap e136: 8 total, 8 up, 8 in 2024-04-25T04:28:39.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:39 smithi078 ceph-mon[62942]: pgmap v244: 97 pgs: 6 peering, 6 active+undersized, 18 active+undersized+degraded, 67 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s wr, 165 op/s; 146/1713 objects degraded (8.523%) 2024-04-25T04:28:39.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:39.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:39.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:39 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 175/1686 objects degraded (10.380%), 21 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:39.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:39 smithi078 ceph-mon[62942]: osdmap e137: 8 total, 8 up, 8 in 2024-04-25T04:28:39.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262522 10.15 MB/sec execute 503 sec latency 1099.313 ms 2024-04-25T04:28:40.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:40 smithi078 ceph-mon[62942]: osdmap e138: 8 total, 8 up, 8 in 2024-04-25T04:28:40.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:40 smithi078 ceph-mon[62942]: pgmap v247: 97 pgs: 10 peering, 2 active+undersized, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.1 MiB/s wr, 708 op/s; 57/1662 objects degraded (3.430%); 4.1 MiB/s, 29 objects/s recovering 2024-04-25T04:28:40.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262623 10.14 MB/sec execute 504 sec latency 956.646 ms 2024-04-25T04:28:40.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:40 smithi112 ceph-mon[50007]: osdmap e138: 8 total, 8 up, 8 in 2024-04-25T04:28:40.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:40 smithi112 ceph-mon[50007]: pgmap v247: 97 pgs: 10 peering, 2 active+undersized, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.1 MiB/s wr, 708 op/s; 57/1662 objects degraded (3.430%); 4.1 MiB/s, 29 objects/s recovering 2024-04-25T04:28:41.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:41 smithi078 ceph-mon[62942]: osdmap e139: 8 total, 8 up, 8 in 2024-04-25T04:28:41.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262941 10.12 MB/sec execute 505 sec latency 47.847 ms 2024-04-25T04:28:41.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:41 smithi112 ceph-mon[50007]: osdmap e139: 8 total, 8 up, 8 in 2024-04-25T04:28:42.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:42 smithi078 ceph-mon[62942]: pgmap v249: 97 pgs: 4 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 637 op/s; 57/1542 objects degraded (3.696%); 4.5 MiB/s, 43 objects/s recovering 2024-04-25T04:28:42.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 263231 10.12 MB/sec execute 506 sec latency 58.857 ms 2024-04-25T04:28:42.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:42 smithi112 ceph-mon[50007]: pgmap v249: 97 pgs: 4 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 637 op/s; 57/1542 objects degraded (3.696%); 4.5 MiB/s, 43 objects/s recovering 2024-04-25T04:28:43.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:43 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 57/1542 objects degraded (3.696%), 2 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:43.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 263536 10.11 MB/sec execute 507 sec latency 73.713 ms 2024-04-25T04:28:43.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:43 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 57/1542 objects degraded (3.696%), 2 pgs degraded (PG_DEGRADED) 2024-04-25T04:28:44.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:44 smithi078 ceph-mon[62942]: pgmap v250: 97 pgs: 2 peering, 95 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.1 MiB/s wr, 475 op/s; 3.2 MiB/s, 38 objects/s recovering 2024-04-25T04:28:44.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 263903 10.11 MB/sec execute 508 sec latency 37.417 ms 2024-04-25T04:28:44.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:44 smithi112 ceph-mon[50007]: pgmap v250: 97 pgs: 2 peering, 95 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.1 MiB/s wr, 475 op/s; 3.2 MiB/s, 38 objects/s recovering 2024-04-25T04:28:45.155 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:28:45.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:45 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/1542 objects degraded (3.696%), 2 pgs degraded) 2024-04-25T04:28:45.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:45 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:28:45.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:45 smithi078 ceph-mon[62942]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:45.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 264346 10.12 MB/sec execute 509 sec latency 38.907 ms 2024-04-25T04:28:45.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:45 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/1542 objects degraded (3.696%), 2 pgs degraded) 2024-04-25T04:28:45.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:45 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:28:45.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:45 smithi112 ceph-mon[50007]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 94s ago 17m 24.6M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 94s ago 17m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (4m) 18s ago 16m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 94s ago 16m 55.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (11m) 94s ago 11m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (11m) 18s ago 11m 15.3M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (7m) 94s ago 18m 510M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (6m) 18s ago 16m 452M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:28:46.225 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (6m) 94s ago 18m 49.9M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (5m) 18s ago 16m 42.7M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (10m) 94s ago 10m 279M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (10m) 18s ago 10m 88.0M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 94s ago 16m 17.5M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (7m) 18s ago 16m 16.9M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 94s ago 15m 358M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 94s ago 14m 286M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 94s ago 14m 190M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (105s) 94s ago 14m 13.7M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (60s) 18s ago 13m 262M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (23s) 18s ago 13m 12.7M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (12m) 18s ago 12m 552M 2704M 17.2.0 e1d6a67b021e 2eb3b069f71e 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (12m) 18s ago 12m 596M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:28:46.226 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 94s ago 15m 73.3M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:28:46.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:46 smithi078 ceph-mon[62942]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:46.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:46 smithi078 ceph-mon[62942]: pgmap v251: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.5 MiB/s wr, 502 op/s; 2.6 MiB/s, 34 objects/s recovering 2024-04-25T04:28:46.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 264748 10.12 MB/sec execute 510 sec latency 38.261 ms 2024-04-25T04:28:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:46 smithi112 ceph-mon[50007]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:46.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:46 smithi112 ceph-mon[50007]: pgmap v251: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.5 MiB/s wr, 502 op/s; 2.6 MiB/s, 34 objects/s recovering 2024-04-25T04:28:46.958 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:28:46.958 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 6 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-25T04:28:46.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 10 2024-04-25T04:28:46.960 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:28:46.960 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:28:47.508 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:28:47.508 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "12/23 daemons upgraded", 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:28:47.509 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:28:47.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:47 smithi078 ceph-mon[62942]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:47.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:47 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2469731163' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:47.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265047 10.12 MB/sec execute 511 sec latency 74.410 ms 2024-04-25T04:28:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:47 smithi112 ceph-mon[50007]: from='client.44325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:47.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:47 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2469731163' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:28:48.134 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:28:48.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:48 smithi078 ceph-mon[62942]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:48.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:48 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3757333615' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:28:48.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:48 smithi078 ceph-mon[62942]: pgmap v252: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.3 MiB/s wr, 459 op/s; 1.4 MiB/s, 24 objects/s recovering 2024-04-25T04:28:48.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:48 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265314 10.12 MB/sec execute 512 sec latency 47.226 ms 2024-04-25T04:28:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:48 smithi112 ceph-mon[50007]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:28:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:48 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3757333615' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:28:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:48 smithi112 ceph-mon[50007]: pgmap v252: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.3 MiB/s wr, 459 op/s; 1.4 MiB/s, 24 objects/s recovering 2024-04-25T04:28:48.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:48 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:49.548 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:49 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:49.549 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:49 smithi112 ceph-mon[50007]: Upgrade: osd.6 is safe to restart 2024-04-25T04:28:49.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:49 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-25T04:28:49.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:49 smithi078 ceph-mon[62942]: Upgrade: osd.6 is safe to restart 2024-04-25T04:28:49.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265653 10.12 MB/sec execute 513 sec latency 56.049 ms 2024-04-25T04:28:50.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:50 smithi078 ceph-mon[62942]: pgmap v253: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.2 MiB/s wr, 321 op/s; 358 KiB/s, 14 objects/s recovering 2024-04-25T04:28:50.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:50.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T04:28:50.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:28:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265948 10.11 MB/sec execute 514 sec latency 45.550 ms 2024-04-25T04:28:50.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:50 smithi112 ceph-mon[50007]: pgmap v253: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.2 MiB/s wr, 321 op/s; 358 KiB/s, 14 objects/s recovering 2024-04-25T04:28:50.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:50.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-25T04:28:50.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:28:51.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:51 smithi112 ceph-mon[50007]: Upgrade: Updating osd.6 2024-04-25T04:28:51.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:51 smithi112 ceph-mon[50007]: Deploying daemon osd.6 on smithi112 2024-04-25T04:28:51.738 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:51 smithi078 ceph-mon[62942]: Upgrade: Updating osd.6 2024-04-25T04:28:51.738 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:51 smithi078 ceph-mon[62942]: Deploying daemon osd.6 on smithi112 2024-04-25T04:28:51.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 266197 10.11 MB/sec execute 515 sec latency 58.389 ms 2024-04-25T04:28:52.637 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:52 smithi112 ceph-mon[50007]: pgmap v254: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.0 MiB/s wr, 420 op/s; 306 KiB/s, 12 objects/s recovering 2024-04-25T04:28:52.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:52 smithi078 ceph-mon[62942]: pgmap v254: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.0 MiB/s wr, 420 op/s; 306 KiB/s, 12 objects/s recovering 2024-04-25T04:28:52.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 266496 10.09 MB/sec execute 516 sec latency 62.351 ms 2024-04-25T04:28:53.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 266814 10.09 MB/sec execute 517 sec latency 44.541 ms 2024-04-25T04:28:54.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:54.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:54.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:54.507 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:54.508 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:28:54.508 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:28:54.508 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:54 smithi112 systemd[1]: Stopping Ceph osd.6 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:28:54.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 267197 10.08 MB/sec execute 518 sec latency 56.982 ms 2024-04-25T04:28:54.900 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:54 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[34412]: 2024-04-25T04:28:54.501+0000 7fb173c28700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:28:54.900 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:54 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[34412]: 2024-04-25T04:28:54.501+0000 7fb173c28700 -1 osd.6 139 *** Got signal Terminated *** 2024-04-25T04:28:54.900 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:54 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[34412]: 2024-04-25T04:28:54.501+0000 7fb173c28700 -1 osd.6 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:28:55.327 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:55 smithi078 ceph-mon[62942]: pgmap v255: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 515 op/s; 133 KiB/s, 7 objects/s recovering 2024-04-25T04:28:55.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:55 smithi078 ceph-mon[62942]: osd.6 marked itself down and dead 2024-04-25T04:28:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:55 smithi112 ceph-mon[50007]: pgmap v255: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 515 op/s; 133 KiB/s, 7 objects/s recovering 2024-04-25T04:28:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:55 smithi112 ceph-mon[50007]: osd.6 marked itself down and dead 2024-04-25T04:28:55.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 267406 10.08 MB/sec execute 519 sec latency 365.798 ms 2024-04-25T04:28:56.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:56 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:28:56.466 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:56 smithi112 ceph-mon[50007]: osdmap e140: 8 total, 7 up, 8 in 2024-04-25T04:28:56.466 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:56 smithi112 podman[59041]: 2024-04-25 04:28:56.147159138 +0000 UTC m=+1.817360004 container died 2eb3b069f71e56e4058aedbff4cef3ba24ade8443e69b977bdbeb89c9064a831 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, release=754, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, RELEASE=HEAD) 2024-04-25T04:28:56.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:56 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:28:56.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:56 smithi078 ceph-mon[62942]: osdmap e140: 8 total, 7 up, 8 in 2024-04-25T04:28:56.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 267499 10.07 MB/sec execute 520 sec latency 1058.320 ms 2024-04-25T04:28:56.892 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:56 smithi112 podman[59041]: 2024-04-25 04:28:56.660865421 +0000 UTC m=+2.331066318 container cleanup 2eb3b069f71e56e4058aedbff4cef3ba24ade8443e69b977bdbeb89c9064a831 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, version=8, ceph=True, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64) 2024-04-25T04:28:56.892 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:56 smithi112 bash[59041]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6 2024-04-25T04:28:57.150 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:56 smithi112 podman[59059]: 2024-04-25 04:28:56.887483756 +0000 UTC m=+0.744326338 container remove 2eb3b069f71e56e4058aedbff4cef3ba24ade8443e69b977bdbeb89c9064a831 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, maintainer=Guillaume Abrioux , version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-04-25T04:28:57.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:57 smithi078 ceph-mon[62942]: pgmap v257: 97 pgs: 10 stale+active+clean, 87 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 486 op/s 2024-04-25T04:28:57.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:57 smithi078 ceph-mon[62942]: osdmap e141: 8 total, 7 up, 8 in 2024-04-25T04:28:57.524 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:57 smithi112 ceph-mon[50007]: pgmap v257: 97 pgs: 10 stale+active+clean, 87 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 486 op/s 2024-04-25T04:28:57.524 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:57 smithi112 ceph-mon[50007]: osdmap e141: 8 total, 7 up, 8 in 2024-04-25T04:28:57.524 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:57 smithi112 podman[59148]: 2024-04-25 04:28:57.276052468 +0000 UTC m=+0.019411590 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:28:57.524 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:57 smithi112 podman[59148]: 2024-04-25 04:28:57.405630958 +0000 UTC m=+0.148990053 container create f4bb9ac6f5b01d2d150fc0a53b2f2e04ff3ac393349d70c11a9a652d378a767b (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:28:57.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 267875 10.07 MB/sec execute 521 sec latency 59.946 ms 2024-04-25T04:28:57.900 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:57 smithi112 podman[59148]: 2024-04-25 04:28:57.650183328 +0000 UTC m=+0.393542422 container init f4bb9ac6f5b01d2d150fc0a53b2f2e04ff3ac393349d70c11a9a652d378a767b (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-25T04:28:57.900 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:57 smithi112 podman[59148]: 2024-04-25 04:28:57.653421416 +0000 UTC m=+0.396780520 container start f4bb9ac6f5b01d2d150fc0a53b2f2e04ff3ac393349d70c11a9a652d378a767b (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-25T04:28:57.900 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:57 smithi112 podman[59148]: 2024-04-25 04:28:57.750450405 +0000 UTC m=+0.493809510 container attach f4bb9ac6f5b01d2d150fc0a53b2f2e04ff3ac393349d70c11a9a652d378a767b (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-25T04:28:58.246 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:57 smithi112 podman[59148]: 2024-04-25 04:28:57.985065899 +0000 UTC m=+0.728425016 container died f4bb9ac6f5b01d2d150fc0a53b2f2e04ff3ac393349d70c11a9a652d378a767b (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-25T04:28:58.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:58 smithi078 ceph-mon[62942]: pgmap v259: 97 pgs: 22 peering, 4 stale+active+clean, 71 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 501 op/s 2024-04-25T04:28:58.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:28:58 smithi078 ceph-mon[62942]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:28:58.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:58 smithi112 ceph-mon[50007]: pgmap v259: 97 pgs: 22 peering, 4 stale+active+clean, 71 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 501 op/s 2024-04-25T04:28:58.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:28:58 smithi112 ceph-mon[50007]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:28:58.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 268200 10.07 MB/sec execute 522 sec latency 55.245 ms 2024-04-25T04:28:59.453 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 podman[59174]: 2024-04-25 04:28:59.198127668 +0000 UTC m=+1.202927567 container remove f4bb9ac6f5b01d2d150fc0a53b2f2e04ff3ac393349d70c11a9a652d378a767b (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-25T04:28:59.453 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-25T04:28:59.453 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 systemd[1]: Stopped Ceph osd.6 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:28:59.453 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.6.service: Consumed 4min 58.497s CPU time. 2024-04-25T04:28:59.737 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 systemd[1]: Starting Ceph osd.6 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:28:59.737 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 podman[59273]: 2024-04-25 04:28:59.593160303 +0000 UTC m=+0.019976908 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:28:59.737 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 podman[59273]: 2024-04-25 04:28:59.732648015 +0000 UTC m=+0.159464605 container create c3e5b72fe25e431f59ad2a0ceaf4ba199235ea40000d2a080bc012979e71a74a (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-04-25T04:28:59.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 268548 10.07 MB/sec execute 523 sec latency 43.465 ms 2024-04-25T04:29:00.255 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 podman[59273]: 2024-04-25 04:28:59.99407995 +0000 UTC m=+0.420896552 container init c3e5b72fe25e431f59ad2a0ceaf4ba199235ea40000d2a080bc012979e71a74a (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True) 2024-04-25T04:29:00.256 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:28:59 smithi112 podman[59273]: 2024-04-25 04:28:59.99762916 +0000 UTC m=+0.424445752 container start c3e5b72fe25e431f59ad2a0ceaf4ba199235ea40000d2a080bc012979e71a74a (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-04-25T04:29:00.256 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 podman[59273]: 2024-04-25 04:29:00.09420216 +0000 UTC m=+0.521018769 container attach c3e5b72fe25e431f59ad2a0ceaf4ba199235ea40000d2a080bc012979e71a74a (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:29:00.382 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:00 smithi078 ceph-mon[62942]: pgmap v260: 97 pgs: 2 active+undersized, 11 active+undersized+degraded, 22 peering, 62 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 347 op/s; 92/1644 objects degraded (5.596%) 2024-04-25T04:29:00.382 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:00 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 92/1644 objects degraded (5.596%), 11 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:00.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-mon[50007]: pgmap v260: 97 pgs: 2 active+undersized, 11 active+undersized+degraded, 22 peering, 62 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 347 op/s; 92/1644 objects degraded (5.596%) 2024-04-25T04:29:00.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 92/1644 objects degraded (5.596%), 11 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:00.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 268797 10.07 MB/sec execute 524 sec latency 44.102 ms 2024-04-25T04:29:00.900 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:00.901 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:00.901 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:00.901 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:00.901 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-25T04:29:00.901 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-25T04:29:00.901 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-25T04:29:00.902 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-25T04:29:00.902 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-25T04:29:00.902 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-25T04:29:00.902 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 bash[59273]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-25T04:29:00.902 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-25T04:29:00.903 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-25T04:29:00.903 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-25T04:29:00.903 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-25T04:29:00.903 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-25T04:29:00.903 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-25T04:29:00.903 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate[59287]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-25T04:29:00.904 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:00 smithi112 podman[59273]: 2024-04-25 04:29:00.615955459 +0000 UTC m=+1.042772040 container died c3e5b72fe25e431f59ad2a0ceaf4ba199235ea40000d2a080bc012979e71a74a (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-25T04:29:01.505 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 podman[59414]: 2024-04-25 04:29:01.241003895 +0000 UTC m=+0.617681968 container remove c3e5b72fe25e431f59ad2a0ceaf4ba199235ea40000d2a080bc012979e71a74a (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-25T04:29:01.505 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 podman[59471]: 2024-04-25 04:29:01.371442043 +0000 UTC m=+0.021390034 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:29:01.766 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 podman[59471]: 2024-04-25 04:29:01.500435591 +0000 UTC m=+0.150383558 container create 75c18bf56ed69bb377517cf7feaef6809b778c230a0e7e88329f62760289942c (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:29:01.766 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 podman[59471]: 2024-04-25 04:29:01.736648643 +0000 UTC m=+0.386596607 container init 75c18bf56ed69bb377517cf7feaef6809b778c230a0e7e88329f62760289942c (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-25T04:29:01.766 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 podman[59471]: 2024-04-25 04:29:01.739811563 +0000 UTC m=+0.389759528 container start 75c18bf56ed69bb377517cf7feaef6809b778c230a0e7e88329f62760289942c (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:29:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: 5 269103 10.06 MB/sec execute 525 sec latency 45.514 ms 2024-04-25T04:29:02.150 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 ceph-osd[59489]: -- 172.21.15.112:0/2619302538 <== mon.1 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561461e13680 con 0x561461ddd000 2024-04-25T04:29:02.150 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 bash[59471]: 75c18bf56ed69bb377517cf7feaef6809b778c230a0e7e88329f62760289942c 2024-04-25T04:29:02.151 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:01 smithi112 systemd[1]: Started Ceph osd.6 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:29:02.650 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:02 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:29:02.279+0000 7f9edecfa7c0 -1 Falling back to public interface 2024-04-25T04:29:02.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 269427 10.06 MB/sec execute 526 sec latency 42.514 ms 2024-04-25T04:29:02.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:02.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:02.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:02.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:02 smithi078 ceph-mon[62942]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.4 MiB/s wr, 303 op/s; 200/1641 objects degraded (12.188%); 8.0 KiB/s, 0 objects/s recovering 2024-04-25T04:29:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:03.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:02 smithi112 ceph-mon[50007]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.4 MiB/s wr, 303 op/s; 200/1641 objects degraded (12.188%); 8.0 KiB/s, 0 objects/s recovering 2024-04-25T04:29:03.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 269715 10.06 MB/sec execute 527 sec latency 43.709 ms 2024-04-25T04:29:04.019 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:03 smithi112 ceph-mon[50007]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-25T04:29:04.138 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:03 smithi078 ceph-mon[62942]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-25T04:29:04.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 270018 10.05 MB/sec execute 528 sec latency 51.198 ms 2024-04-25T04:29:05.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:04 smithi112 ceph-mon[50007]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.5 MiB/s wr, 416 op/s; 199/1677 objects degraded (11.866%); 7.4 KiB/s, 0 objects/s recovering 2024-04-25T04:29:05.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:04 smithi078 ceph-mon[62942]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.5 MiB/s wr, 416 op/s; 199/1677 objects degraded (11.866%); 7.4 KiB/s, 0 objects/s recovering 2024-04-25T04:29:05.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 270326 10.04 MB/sec execute 529 sec latency 54.973 ms 2024-04-25T04:29:06.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 270754 10.04 MB/sec execute 530 sec latency 36.981 ms 2024-04-25T04:29:07.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:07 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:07.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:07 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:07.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:07 smithi078 ceph-mon[62942]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 469 op/s; 202/1683 objects degraded (12.002%); 6.4 KiB/s, 0 objects/s recovering 2024-04-25T04:29:07.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:07 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:07.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:07 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:07.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:07 smithi112 ceph-mon[50007]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 469 op/s; 202/1683 objects degraded (12.002%); 6.4 KiB/s, 0 objects/s recovering 2024-04-25T04:29:07.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 271037 10.04 MB/sec execute 531 sec latency 43.878 ms 2024-04-25T04:29:08.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:08 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 202/1683 objects degraded (12.002%), 25 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:08.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:08.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:08.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:08 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 202/1683 objects degraded (12.002%), 25 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:08.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:08.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:08.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 271480 10.05 MB/sec execute 532 sec latency 48.254 ms 2024-04-25T04:29:09.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:09 smithi078 ceph-mon[62942]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 378 op/s; 195/1659 objects degraded (11.754%); 5.5 KiB/s, 0 objects/s recovering 2024-04-25T04:29:09.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:09.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:09.392 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:09 smithi112 ceph-mon[50007]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 378 op/s; 195/1659 objects degraded (11.754%); 5.5 KiB/s, 0 objects/s recovering 2024-04-25T04:29:09.392 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:09.392 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:09.650 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:09 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:29:09.635+0000 7f9edecfa7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-25T04:29:09.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 271884 10.05 MB/sec execute 533 sec latency 42.878 ms 2024-04-25T04:29:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 272266 10.05 MB/sec execute 534 sec latency 42.771 ms 2024-04-25T04:29:11.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:11 smithi078 ceph-mon[62942]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.8 MiB/s wr, 446 op/s; 195/1650 objects degraded (11.818%); 5.3 KiB/s, 0 objects/s recovering 2024-04-25T04:29:11.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:11 smithi078 ceph-mon[62942]: from='osd.6 [v2:172.21.15.112:6816/3951584070,v1:172.21.15.112:6817/3951584070]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:29:11.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:11 smithi078 ceph-mon[62942]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:29:11.400 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:11 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:29:11.087+0000 7f9edecfa7c0 -1 osd.6 139 log_to_monitors true 2024-04-25T04:29:11.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:11 smithi112 ceph-mon[50007]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.8 MiB/s wr, 446 op/s; 195/1650 objects degraded (11.818%); 5.3 KiB/s, 0 objects/s recovering 2024-04-25T04:29:11.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:11 smithi112 ceph-mon[50007]: from='osd.6 [v2:172.21.15.112:6816/3951584070,v1:172.21.15.112:6817/3951584070]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:29:11.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:11 smithi112 ceph-mon[50007]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-25T04:29:11.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 272504 10.05 MB/sec execute 535 sec latency 238.966 ms 2024-04-25T04:29:12.150 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:29:11 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:29:11.684+0000 7f9ecc14a700 -1 osd.6 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:29:12.389 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: osdmap e142: 8 total, 7 up, 8 in 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='osd.6 [v2:172.21.15.112:6816/3951584070,v1:172.21.15.112:6817/3951584070]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='osd.6 ' entity='osd.6' 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:12.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:12.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:12.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:12.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:12 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-25T04:29:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: osdmap e142: 8 total, 7 up, 8 in 2024-04-25T04:29:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='osd.6 [v2:172.21.15.112:6816/3951584070,v1:172.21.15.112:6817/3951584070]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:29:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:29:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='osd.6 ' entity='osd.6' 2024-04-25T04:29:12.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:12.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:12 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:12.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 272926 10.05 MB/sec execute 536 sec latency 137.896 ms 2024-04-25T04:29:13.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:13.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-25T04:29:13.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 535 op/s; 203/1689 objects degraded (12.019%) 2024-04-25T04:29:13.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:29:13.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: osd.6 [v2:172.21.15.112:6816/3951584070,v1:172.21.15.112:6817/3951584070] boot 2024-04-25T04:29:13.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: osdmap e143: 8 total, 8 up, 8 in 2024-04-25T04:29:13.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:29:13.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:13 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 203/1689 objects degraded (12.019%), 25 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 535 op/s; 203/1689 objects degraded (12.019%) 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: osd.6 [v2:172.21.15.112:6816/3951584070,v1:172.21.15.112:6817/3951584070] boot 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: osdmap e143: 8 total, 8 up, 8 in 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-25T04:29:13.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:13 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 203/1689 objects degraded (12.019%), 25 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:13.868 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273024 10.03 MB/sec execute 537 sec latency 1137.925 ms 2024-04-25T04:29:14.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:14 smithi078 ceph-mon[62942]: osdmap e144: 8 total, 8 up, 8 in 2024-04-25T04:29:14.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:14 smithi078 ceph-mon[62942]: pgmap v270: 97 pgs: 1 remapped+peering, 10 active+undersized, 21 active+undersized+degraded, 65 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.8 MiB/s wr, 455 op/s; 169/1716 objects degraded (9.848%); 0 B/s, 3 objects/s recovering 2024-04-25T04:29:14.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273059 10.02 MB/sec execute 538 sec latency 2137.958 ms 2024-04-25T04:29:14.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:14 smithi112 ceph-mon[50007]: osdmap e144: 8 total, 8 up, 8 in 2024-04-25T04:29:14.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:14 smithi112 ceph-mon[50007]: pgmap v270: 97 pgs: 1 remapped+peering, 10 active+undersized, 21 active+undersized+degraded, 65 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.8 MiB/s wr, 455 op/s; 169/1716 objects degraded (9.848%); 0 B/s, 3 objects/s recovering 2024-04-25T04:29:15.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:15 smithi078 ceph-mon[62942]: osdmap e145: 8 total, 8 up, 8 in 2024-04-25T04:29:15.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273328 10.01 MB/sec execute 539 sec latency 2326.749 ms 2024-04-25T04:29:15.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:15 smithi112 ceph-mon[50007]: osdmap e145: 8 total, 8 up, 8 in 2024-04-25T04:29:16.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:16 smithi078 ceph-mon[62942]: osdmap e146: 8 total, 8 up, 8 in 2024-04-25T04:29:16.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:16 smithi078 ceph-mon[62942]: pgmap v273: 97 pgs: 1 remapped+peering, 9 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 16 active+undersized+degraded, 69 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 266 op/s; 152/1698 objects degraded (8.952%); 1.0 MiB/s, 10 objects/s recovering 2024-04-25T04:29:16.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273588 10.00 MB/sec execute 540 sec latency 1004.832 ms 2024-04-25T04:29:16.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:16 smithi112 ceph-mon[50007]: osdmap e146: 8 total, 8 up, 8 in 2024-04-25T04:29:16.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:16 smithi112 ceph-mon[50007]: pgmap v273: 97 pgs: 1 remapped+peering, 9 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 16 active+undersized+degraded, 69 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 266 op/s; 152/1698 objects degraded (8.952%); 1.0 MiB/s, 10 objects/s recovering 2024-04-25T04:29:17.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:17 smithi078 ceph-mon[62942]: osdmap e147: 8 total, 8 up, 8 in 2024-04-25T04:29:17.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273928 10.00 MB/sec execute 541 sec latency 43.545 ms 2024-04-25T04:29:17.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:17 smithi112 ceph-mon[50007]: osdmap e147: 8 total, 8 up, 8 in 2024-04-25T04:29:18.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:18 smithi078 ceph-mon[62942]: osdmap e148: 8 total, 8 up, 8 in 2024-04-25T04:29:18.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:18 smithi078 ceph-mon[62942]: pgmap v276: 97 pgs: 4 peering, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.9 MiB/s wr, 450 op/s; 29/1644 objects degraded (1.764%); 2.8 MiB/s, 74 keys/s, 38 objects/s recovering 2024-04-25T04:29:18.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:18 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 152/1698 objects degraded (8.952%), 17 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:18.665 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:29:18.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 274294 10.00 MB/sec execute 542 sec latency 69.119 ms 2024-04-25T04:29:18.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:18 smithi112 ceph-mon[50007]: osdmap e148: 8 total, 8 up, 8 in 2024-04-25T04:29:18.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:18 smithi112 ceph-mon[50007]: pgmap v276: 97 pgs: 4 peering, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.9 MiB/s wr, 450 op/s; 29/1644 objects degraded (1.764%); 2.8 MiB/s, 74 keys/s, 38 objects/s recovering 2024-04-25T04:29:18.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:18 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 152/1698 objects degraded (8.952%), 17 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:19.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:19 smithi078 ceph-mon[62942]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:19.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:19 smithi078 ceph-mon[62942]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 2m ago 17m 24.6M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 17m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (5m) 13s ago 17m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (8m) 2m ago 17m 55.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (11m) 2m ago 11m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (11m) 13s ago 11m 15.4M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:29:19.708 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (8m) 2m ago 18m 510M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (7m) 13s ago 16m 453M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (6m) 2m ago 19m 49.9M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (6m) 13s ago 16m 42.2M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (11m) 13s ago 11m 88.0M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 17m 17.5M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (8m) 13s ago 16m 16.5M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 16m 358M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 2m ago 15m 286M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 15m 190M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 14m 13.7M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (94s) 13s ago 14m 291M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (56s) 13s ago 13m 271M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (18s) 13s ago 13m 15.1M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:29:19.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (13m) 13s ago 13m 649M 2704M 17.2.0 e1d6a67b021e 464cd6396efa 2024-04-25T04:29:19.710 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 16m 73.3M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:29:19.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 274630 9.99 MB/sec execute 543 sec latency 40.392 ms 2024-04-25T04:29:19.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:19 smithi112 ceph-mon[50007]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:19.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:19 smithi112 ceph-mon[50007]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:20.246 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:29:20.246 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:29:20.246 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 7 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:20.247 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:29:20.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:29:20.248 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:20.248 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:29:20.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-25T04:29:20.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 11 2024-04-25T04:29:20.248 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:29:20.248 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:29:20.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:20 smithi078 ceph-mon[62942]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:20.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:20 smithi078 ceph-mon[62942]: pgmap v277: 97 pgs: 4 peering, 93 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.2 MiB/s wr, 387 op/s; 2.1 MiB/s, 52 keys/s, 36 objects/s recovering 2024-04-25T04:29:20.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:20 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/445362940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:20.817 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:20 smithi112 ceph-mon[50007]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:20.817 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:20 smithi112 ceph-mon[50007]: pgmap v277: 97 pgs: 4 peering, 93 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.2 MiB/s wr, 387 op/s; 2.1 MiB/s, 52 keys/s, 36 objects/s recovering 2024-04-25T04:29:20.817 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:20 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/445362940' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:29:20.837 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:29:20.838 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:29:20.838 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "13/23 daemons upgraded", 2024-04-25T04:29:20.838 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-04-25T04:29:20.838 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:29:20.838 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:29:20.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 274915 9.99 MB/sec execute 544 sec latency 51.129 ms 2024-04-25T04:29:21.478 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:29:21.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:21 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1644 objects degraded (1.764%), 1 pg degraded) 2024-04-25T04:29:21.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:21 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:29:21.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:21 smithi078 ceph-mon[62942]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:21.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:21 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2921963689' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:29:21.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 275323 10.00 MB/sec execute 545 sec latency 52.184 ms 2024-04-25T04:29:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:21 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 29/1644 objects degraded (1.764%), 1 pg degraded) 2024-04-25T04:29:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:21 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:29:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:21 smithi112 ceph-mon[50007]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:21.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:21 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2921963689' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:29:22.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:22 smithi078 ceph-mon[62942]: pgmap v278: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.1 MiB/s wr, 507 op/s; 1.2 MiB/s, 44 keys/s, 27 objects/s recovering 2024-04-25T04:29:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: 5 275736 10.00 MB/sec execute 546 sec latency 35.060 ms 2024-04-25T04:29:22.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:22 smithi112 ceph-mon[50007]: pgmap v278: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.1 MiB/s wr, 507 op/s; 1.2 MiB/s, 44 keys/s, 27 objects/s recovering 2024-04-25T04:29:23.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 276048 10.00 MB/sec execute 547 sec latency 51.317 ms 2024-04-25T04:29:24.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:24.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:24.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:24.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:24.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 276321 10.00 MB/sec execute 548 sec latency 39.518 ms 2024-04-25T04:29:25.288 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:25 smithi078 ceph-mon[62942]: pgmap v279: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.6 MiB/s wr, 466 op/s; 986 KiB/s, 37 keys/s, 22 objects/s recovering 2024-04-25T04:29:25.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:25 smithi112 ceph-mon[50007]: pgmap v279: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.6 MiB/s wr, 466 op/s; 986 KiB/s, 37 keys/s, 22 objects/s recovering 2024-04-25T04:29:25.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 276588 9.99 MB/sec execute 549 sec latency 44.777 ms 2024-04-25T04:29:26.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 276895 9.99 MB/sec execute 550 sec latency 55.024 ms 2024-04-25T04:29:27.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:27 smithi078 ceph-mon[62942]: pgmap v280: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.3 MiB/s wr, 296 op/s; 68 KiB/s, 5 objects/s recovering 2024-04-25T04:29:27.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:27 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:27.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:27 smithi112 ceph-mon[50007]: pgmap v280: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.3 MiB/s wr, 296 op/s; 68 KiB/s, 5 objects/s recovering 2024-04-25T04:29:27.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:27.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 277198 9.98 MB/sec execute 551 sec latency 73.756 ms 2024-04-25T04:29:28.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:28 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:28.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:28 smithi078 ceph-mon[62942]: Upgrade: osd.7 is safe to restart 2024-04-25T04:29:28.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:28 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-25T04:29:28.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:28 smithi112 ceph-mon[50007]: Upgrade: osd.7 is safe to restart 2024-04-25T04:29:28.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 277492 9.97 MB/sec execute 552 sec latency 63.648 ms 2024-04-25T04:29:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:29 smithi078 ceph-mon[62942]: pgmap v281: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 450 op/s; 62 KiB/s, 4 objects/s recovering 2024-04-25T04:29:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:29 smithi078 ceph-mon[62942]: Upgrade: Updating osd.7 2024-04-25T04:29:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T04:29:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:29 smithi078 ceph-mon[62942]: Deploying daemon osd.7 on smithi112 2024-04-25T04:29:29.474 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:29 smithi112 ceph-mon[50007]: pgmap v281: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 450 op/s; 62 KiB/s, 4 objects/s recovering 2024-04-25T04:29:29.474 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:29.474 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:29 smithi112 ceph-mon[50007]: Upgrade: Updating osd.7 2024-04-25T04:29:29.474 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:29.474 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-25T04:29:29.475 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:29.475 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:29 smithi112 ceph-mon[50007]: Deploying daemon osd.7 on smithi112 2024-04-25T04:29:29.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 277892 9.97 MB/sec execute 553 sec latency 69.557 ms 2024-04-25T04:29:30.417 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:30 smithi078 ceph-mon[62942]: pgmap v282: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 55 KiB/s, 4 objects/s recovering 2024-04-25T04:29:30.503 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:30 smithi112 ceph-mon[50007]: pgmap v282: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 437 op/s; 55 KiB/s, 4 objects/s recovering 2024-04-25T04:29:30.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 278229 9.96 MB/sec execute 554 sec latency 38.896 ms 2024-04-25T04:29:31.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 278501 9.97 MB/sec execute 555 sec latency 43.924 ms 2024-04-25T04:29:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:32 smithi112 ceph-mon[50007]: pgmap v283: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 563 op/s 2024-04-25T04:29:32.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:32 smithi078 ceph-mon[62942]: pgmap v283: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 563 op/s 2024-04-25T04:29:32.743 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:32 smithi112 systemd[1]: Stopping Ceph osd.7 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:29:32.871 INFO:teuthology.orchestra.run.smithi078.stdout: 5 278878 9.98 MB/sec execute 556 sec latency 45.586 ms 2024-04-25T04:29:33.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[37424]: 2024-04-25T04:29:32.737+0000 7f40df4c9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:29:33.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[37424]: 2024-04-25T04:29:32.737+0000 7f40df4c9700 -1 osd.7 148 *** Got signal Terminated *** 2024-04-25T04:29:33.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[37424]: 2024-04-25T04:29:32.737+0000 7f40df4c9700 -1 osd.7 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:29:33.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:33 smithi078 ceph-mon[62942]: osd.7 marked itself down and dead 2024-04-25T04:29:33.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:33 smithi112 ceph-mon[50007]: osd.7 marked itself down and dead 2024-04-25T04:29:33.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 279014 9.97 MB/sec execute 557 sec latency 613.385 ms 2024-04-25T04:29:34.227 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:33 smithi112 podman[61302]: 2024-04-25 04:29:33.969484124 +0000 UTC m=+1.337355831 container died 464cd6396efa51fc36ad568ddba2e448d054a209c39bcedd5bd4c9f206395702 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_CLEAN=True, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0) 2024-04-25T04:29:34.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:34 smithi078 ceph-mon[62942]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:29:34.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:34 smithi078 ceph-mon[62942]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-25T04:29:34.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:34 smithi078 ceph-mon[62942]: osdmap e149: 8 total, 7 up, 8 in 2024-04-25T04:29:34.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:34 smithi078 ceph-mon[62942]: pgmap v285: 97 pgs: 8 peering, 10 stale+active+clean, 79 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 458 op/s 2024-04-25T04:29:34.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:34 smithi112 ceph-mon[50007]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-25T04:29:34.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:34 smithi112 ceph-mon[50007]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-25T04:29:34.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:34 smithi112 ceph-mon[50007]: osdmap e149: 8 total, 7 up, 8 in 2024-04-25T04:29:34.644 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:34 smithi112 ceph-mon[50007]: pgmap v285: 97 pgs: 8 peering, 10 stale+active+clean, 79 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 458 op/s 2024-04-25T04:29:34.645 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:34 smithi112 podman[61302]: 2024-04-25 04:29:34.329946031 +0000 UTC m=+1.697817746 container cleanup 464cd6396efa51fc36ad568ddba2e448d054a209c39bcedd5bd4c9f206395702 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., release=754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, name=centos-stream, vcs-type=git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-25T04:29:34.645 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:34 smithi112 bash[61302]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7 2024-04-25T04:29:34.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 279260 9.96 MB/sec execute 558 sec latency 1067.221 ms 2024-04-25T04:29:34.900 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:34 smithi112 podman[61320]: 2024-04-25 04:29:34.639907947 +0000 UTC m=+0.673160317 container remove 464cd6396efa51fc36ad568ddba2e448d054a209c39bcedd5bd4c9f206395702 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, distribution-scope=public, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.buildah.version=1.19.8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux ) 2024-04-25T04:29:35.267 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:34 smithi112 podman[61406]: 2024-04-25 04:29:34.879862941 +0000 UTC m=+0.019779910 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:29:35.267 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:34 smithi112 podman[61406]: 2024-04-25 04:29:34.990966175 +0000 UTC m=+0.130883118 container create dae10c6dfa0270344eb537309c3e75f925ae8889add0562f25bc09dc040d066e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-25T04:29:35.267 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:35 smithi112 podman[61406]: 2024-04-25 04:29:35.219115068 +0000 UTC m=+0.359032014 container init dae10c6dfa0270344eb537309c3e75f925ae8889add0562f25bc09dc040d066e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:29:35.267 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:35 smithi112 podman[61406]: 2024-04-25 04:29:35.22267231 +0000 UTC m=+0.362589256 container start dae10c6dfa0270344eb537309c3e75f925ae8889add0562f25bc09dc040d066e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-25T04:29:35.286 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:35 smithi078 ceph-mon[62942]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:29:35.286 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:35 smithi078 ceph-mon[62942]: osdmap e150: 8 total, 7 up, 8 in 2024-04-25T04:29:35.570 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:35 smithi112 ceph-mon[50007]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-25T04:29:35.570 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:35 smithi112 ceph-mon[50007]: osdmap e150: 8 total, 7 up, 8 in 2024-04-25T04:29:35.571 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:35 smithi112 podman[61406]: 2024-04-25 04:29:35.319252591 +0000 UTC m=+0.459169540 container attach dae10c6dfa0270344eb537309c3e75f925ae8889add0562f25bc09dc040d066e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-25T04:29:35.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 279613 9.96 MB/sec execute 559 sec latency 47.541 ms 2024-04-25T04:29:35.899 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:35 smithi112 podman[61406]: 2024-04-25 04:29:35.566335208 +0000 UTC m=+0.706252170 container died dae10c6dfa0270344eb537309c3e75f925ae8889add0562f25bc09dc040d066e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-25T04:29:36.297 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:36 smithi078 ceph-mon[62942]: pgmap v287: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 8 peering, 8 stale+active+clean, 72 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s wr, 325 op/s; 49/1650 objects degraded (2.970%) 2024-04-25T04:29:36.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:36 smithi112 ceph-mon[50007]: pgmap v287: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 8 peering, 8 stale+active+clean, 72 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 MiB/s wr, 325 op/s; 49/1650 objects degraded (2.970%) 2024-04-25T04:29:36.650 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:36 smithi112 podman[61432]: 2024-04-25 04:29:36.215847613 +0000 UTC m=+0.640013856 container remove dae10c6dfa0270344eb537309c3e75f925ae8889add0562f25bc09dc040d066e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:29:36.650 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:36 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-25T04:29:36.650 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:36 smithi112 systemd[1]: Stopped Ceph osd.7 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:29:36.650 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:36 smithi112 systemd[1]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.7.service: Consumed 7min 42.668s CPU time. 2024-04-25T04:29:36.651 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:36 smithi112 systemd[1]: Starting Ceph osd.7 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:29:36.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 279904 9.96 MB/sec execute 560 sec latency 63.026 ms 2024-04-25T04:29:37.024 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:36 smithi112 podman[61540]: 2024-04-25 04:29:36.578054619 +0000 UTC m=+0.020799883 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:29:37.024 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:36 smithi112 podman[61540]: 2024-04-25 04:29:36.725436045 +0000 UTC m=+0.168181297 container create ee13742ff962c48ea185a1e6580ae6b8786661dbc0fe1b5d9514405f9f381e13 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-25T04:29:37.024 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 podman[61540]: 2024-04-25 04:29:37.019978868 +0000 UTC m=+0.462724120 container init ee13742ff962c48ea185a1e6580ae6b8786661dbc0fe1b5d9514405f9f381e13 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-25T04:29:37.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:37 smithi078 ceph-mon[62942]: Health check failed: Degraded data redundancy: 49/1650 objects degraded (2.970%), 7 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:37.333 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-mon[50007]: Health check failed: Degraded data redundancy: 49/1650 objects degraded (2.970%), 7 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:37.333 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 podman[61540]: 2024-04-25 04:29:37.023255893 +0000 UTC m=+0.466001148 container start ee13742ff962c48ea185a1e6580ae6b8786661dbc0fe1b5d9514405f9f381e13 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2) 2024-04-25T04:29:37.594 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 podman[61540]: 2024-04-25 04:29:37.329053658 +0000 UTC m=+0.771798912 container attach ee13742ff962c48ea185a1e6580ae6b8786661dbc0fe1b5d9514405f9f381e13 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424) 2024-04-25T04:29:37.594 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:37.594 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 bash[61540]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-25T04:29:37.595 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-25T04:29:37.596 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate[61555]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-25T04:29:37.596 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:37 smithi112 podman[61540]: 2024-04-25 04:29:37.589493418 +0000 UTC m=+1.032238668 container died ee13742ff962c48ea185a1e6580ae6b8786661dbc0fe1b5d9514405f9f381e13 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:29:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 280250 9.95 MB/sec execute 561 sec latency 61.906 ms 2024-04-25T04:29:38.606 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 podman[61683]: 2024-04-25 04:29:38.342595899 +0000 UTC m=+0.744140272 container remove ee13742ff962c48ea185a1e6580ae6b8786661dbc0fe1b5d9514405f9f381e13 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-25T04:29:38.606 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 podman[61745]: 2024-04-25 04:29:38.473176485 +0000 UTC m=+0.023662690 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:29:38.606 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 podman[61745]: 2024-04-25 04:29:38.601945662 +0000 UTC m=+0.152431828 container create d9186eec4bcc9594a0e33f9fe238db9310b915a88abedbd062ea7664873c1b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-25T04:29:38.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 280609 9.95 MB/sec execute 562 sec latency 66.474 ms 2024-04-25T04:29:39.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 podman[61745]: 2024-04-25 04:29:38.863565256 +0000 UTC m=+0.414051435 container init d9186eec4bcc9594a0e33f9fe238db9310b915a88abedbd062ea7664873c1b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-25T04:29:39.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 podman[61745]: 2024-04-25 04:29:38.867507023 +0000 UTC m=+0.417993194 container start d9186eec4bcc9594a0e33f9fe238db9310b915a88abedbd062ea7664873c1b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-25T04:29:39.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 ceph-osd[61764]: -- 172.21.15.112:0/1431012728 <== mon.1 v2:172.21.15.112:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e3b3ae5680 con 0x55e3b3aaf000 2024-04-25T04:29:39.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 bash[61745]: d9186eec4bcc9594a0e33f9fe238db9310b915a88abedbd062ea7664873c1b09 2024-04-25T04:29:39.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:38 smithi112 systemd[1]: Started Ceph osd.7 for 9a8d9f72-02b9-11ef-bc93-c7b262605968. 2024-04-25T04:29:39.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:39.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:39.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:39 smithi078 ceph-mon[62942]: pgmap v288: 97 pgs: 4 active+undersized, 30 active+undersized+degraded, 8 peering, 55 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 470 op/s; 225/1683 objects degraded (13.369%) 2024-04-25T04:29:39.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:39.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:39.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:39.650 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:29:39.425+0000 7fdabf5f67c0 -1 Falling back to public interface 2024-04-25T04:29:39.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:39.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:39.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:39 smithi112 ceph-mon[50007]: pgmap v288: 97 pgs: 4 active+undersized, 30 active+undersized+degraded, 8 peering, 55 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 470 op/s; 225/1683 objects degraded (13.369%) 2024-04-25T04:29:39.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:39.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:39.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:39.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 280952 9.94 MB/sec execute 563 sec latency 48.166 ms 2024-04-25T04:29:40.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:40 smithi078 ceph-mon[62942]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s wr, 384 op/s; 272/1716 objects degraded (15.851%) 2024-04-25T04:29:40.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:40 smithi078 ceph-mon[62942]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-25T04:29:40.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:40 smithi112 ceph-mon[50007]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s wr, 384 op/s; 272/1716 objects degraded (15.851%) 2024-04-25T04:29:40.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:40 smithi112 ceph-mon[50007]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-04-25T04:29:40.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 281411 9.95 MB/sec execute 564 sec latency 84.176 ms 2024-04-25T04:29:41.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 281835 9.94 MB/sec execute 565 sec latency 36.119 ms 2024-04-25T04:29:42.427 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:42 smithi078 ceph-mon[62942]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 551 op/s; 277/1677 objects degraded (16.518%) 2024-04-25T04:29:42.427 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:42 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 277/1677 objects degraded (16.518%), 35 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:42.527 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:42 smithi112 ceph-mon[50007]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 551 op/s; 277/1677 objects degraded (16.518%) 2024-04-25T04:29:42.528 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:42 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 277/1677 objects degraded (16.518%), 35 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:42.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282123 9.95 MB/sec execute 566 sec latency 56.142 ms 2024-04-25T04:29:43.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282518 9.96 MB/sec execute 567 sec latency 51.136 ms 2024-04-25T04:29:44.203 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:44.203 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:44 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:44 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:44.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282925 9.95 MB/sec execute 568 sec latency 40.964 ms 2024-04-25T04:29:45.208 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:45 smithi112 ceph-mon[50007]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 497 op/s; 282/1710 objects degraded (16.491%) 2024-04-25T04:29:45.209 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:45.209 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:45.230 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:45 smithi078 ceph-mon[62942]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 497 op/s; 282/1710 objects degraded (16.491%) 2024-04-25T04:29:45.230 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:45.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:45.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 283177 9.95 MB/sec execute 569 sec latency 46.602 ms 2024-04-25T04:29:46.508 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:46 smithi078 ceph-mon[62942]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 450 op/s; 276/1695 objects degraded (16.283%) 2024-04-25T04:29:46.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:46 smithi112 ceph-mon[50007]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 450 op/s; 276/1695 objects degraded (16.283%) 2024-04-25T04:29:46.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 283428 9.95 MB/sec execute 570 sec latency 58.108 ms 2024-04-25T04:29:47.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 283676 9.94 MB/sec execute 571 sec latency 54.042 ms 2024-04-25T04:29:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 283976 9.93 MB/sec execute 572 sec latency 72.003 ms 2024-04-25T04:29:49.150 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:29:48.832+0000 7fdabf5f67c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-25T04:29:49.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 575 op/s; 277/1722 objects degraded (16.086%) 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-25T04:29:49.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-25T04:29:49.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-25T04:29:49.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-25T04:29:49.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-25T04:29:49.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:49 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 276/1695 objects degraded (16.283%), 35 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:29:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 575 op/s; 277/1722 objects degraded (16.086%) 2024-04-25T04:29:49.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-25T04:29:49.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-25T04:29:49.652 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:49 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 276/1695 objects degraded (16.283%), 35 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:49.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 284274 9.93 MB/sec execute 573 sec latency 55.138 ms 2024-04-25T04:29:50.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:50 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all osd 2024-04-25T04:29:50.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:50 smithi078 ceph-mon[62942]: Upgrade: Setting require_osd_release to 19 squid 2024-04-25T04:29:50.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:50 smithi078 ceph-mon[62942]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-25T04:29:50.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-25T04:29:50.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:50 smithi078 ceph-mon[62942]: osdmap e151: 8 total, 7 up, 8 in 2024-04-25T04:29:50.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:50 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.nlrtle"]}]: dispatch 2024-04-25T04:29:50.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:50 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all osd 2024-04-25T04:29:50.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:50 smithi112 ceph-mon[50007]: Upgrade: Setting require_osd_release to 19 squid 2024-04-25T04:29:50.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:50 smithi112 ceph-mon[50007]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-25T04:29:50.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-25T04:29:50.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:50 smithi112 ceph-mon[50007]: osdmap e151: 8 total, 7 up, 8 in 2024-04-25T04:29:50.554 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:50 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.nlrtle"]}]: dispatch 2024-04-25T04:29:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 284535 9.92 MB/sec execute 574 sec latency 66.900 ms 2024-04-25T04:29:50.899 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:50 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:29:50.787+0000 7fdabf5f67c0 -1 osd.7 148 log_to_monitors true 2024-04-25T04:29:51.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:51 smithi078 ceph-mon[62942]: Upgrade: It appears safe to stop mds.foofs.smithi078.nlrtle 2024-04-25T04:29:51.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:51 smithi078 ceph-mon[62942]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 433 op/s; 271/1713 objects degraded (15.820%) 2024-04-25T04:29:51.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:51 smithi078 ceph-mon[62942]: from='osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T04:29:51.566 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:29:51 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:29:51.232+0000 7fdab0a4e700 -1 osd.7 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-25T04:29:51.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:51 smithi112 ceph-mon[50007]: Upgrade: It appears safe to stop mds.foofs.smithi078.nlrtle 2024-04-25T04:29:51.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:51 smithi112 ceph-mon[50007]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.8 MiB/s wr, 433 op/s; 271/1713 objects degraded (15.820%) 2024-04-25T04:29:51.567 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:51 smithi112 ceph-mon[50007]: from='osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-25T04:29:51.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 284939 9.92 MB/sec execute 575 sec latency 57.950 ms 2024-04-25T04:29:51.969 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:29:52.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:52 smithi078 ceph-mon[62942]: from='osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-25T04:29:52.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:52 smithi078 ceph-mon[62942]: osdmap e152: 8 total, 7 up, 8 in 2024-04-25T04:29:52.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:52 smithi078 ceph-mon[62942]: from='osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:29:52.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:52 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:52.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:52 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:29:52.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:52 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:52.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:52 smithi112 ceph-mon[50007]: from='osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298]' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-25T04:29:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:52 smithi112 ceph-mon[50007]: osdmap e152: 8 total, 7 up, 8 in 2024-04-25T04:29:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:52 smithi112 ceph-mon[50007]: from='osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi112", "root=default"]}]: dispatch 2024-04-25T04:29:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:52 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:52 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.nlrtle", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:29:52.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:52 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:29:52.774 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:29:52.774 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 2m ago 18m 24.6M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:29:52.774 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 2m ago 18m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:29:52.774 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (5m) 9s ago 17m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (8m) 2m ago 17m 55.0M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (12m) 2m ago 12m 40.6M - 17.2.0 e1d6a67b021e 0cdb495f2d0c 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (12m) 9s ago 12m 15.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (8m) 2m ago 19m 510M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (7m) 9s ago 17m 453M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 19m 49.9M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (6m) 9s ago 17m 44.0M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (11m) 9s ago 11m 88.0M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 17m 17.5M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (8m) 9s ago 17m 18.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 2m ago 16m 358M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 16m 286M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 15m 190M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:29:52.775 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 15m 13.7M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:29:52.776 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (2m) 9s ago 14m 323M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:29:52.776 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (89s) 9s ago 14m 293M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:29:52.776 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (51s) 9s ago 14m 283M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:29:52.776 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (14s) 9s ago 13m 13.6M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:29:52.776 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 17m 73.3M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:29:52.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285075 9.91 MB/sec execute 576 sec latency 692.860 ms 2024-04-25T04:29:53.183 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:29:53.183 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:29:53.183 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:29:53.183 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:53.183 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 12 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:29:53.184 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:29:53.563 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:29:53.563 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:29:53.563 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:29:53.563 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:29:53.563 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "14/23 daemons upgraded", 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:29:53.564 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:29:53.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:53.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: Upgrade: Updating mds.foofs.smithi078.nlrtle 2024-04-25T04:29:53.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: Deploying daemon mds.foofs.smithi078.nlrtle on smithi078 2024-04-25T04:29:53.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 568 op/s; 253/1602 objects degraded (15.793%) 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298] boot 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: osdmap e153: 8 total, 8 up, 8 in 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: from='client.44371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/879045118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:53.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: Upgrade: Updating mds.foofs.smithi078.nlrtle 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: Deploying daemon mds.foofs.smithi078.nlrtle on smithi078 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 568 op/s; 253/1602 objects degraded (15.793%) 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: osd.7 [v2:172.21.15.112:6824/1776310298,v1:172.21.15.112:6825/1776310298] boot 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: osdmap e153: 8 total, 8 up, 8 in 2024-04-25T04:29:53.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-25T04:29:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: from='client.44371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/879045118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:29:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:29:53.651 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:29:53.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285075 9.89 MB/sec execute 577 sec latency 1692.914 ms 2024-04-25T04:29:53.972 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 253/1602 objects degraded (15.793%), 35 pgs degraded 2024-04-25T04:29:53.972 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 253/1602 objects degraded (15.793%), 35 pgs degraded 2024-04-25T04:29:53.972 INFO:teuthology.orchestra.run.smithi078.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-04-25T04:29:53.972 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-04-25T04:29:53.972 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-04-25T04:29:53.972 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-04-25T04:29:53.973 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-04-25T04:29:53.974 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-04-25T04:29:53.975 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-04-25T04:29:54.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:54 smithi078 ceph-mon[62942]: osdmap e154: 8 total, 8 up, 8 in 2024-04-25T04:29:54.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:54 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 253/1602 objects degraded (15.793%), 35 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:54.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:54 smithi078 ceph-mon[62942]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:54.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:54 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3603056724' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:29:54.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:54 smithi078 ceph-mon[62942]: pgmap v300: 97 pgs: 1 activating, 3 remapped+peering, 4 active+undersized, 32 active+undersized+degraded, 57 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 526 op/s; 216/1596 objects degraded (13.534%) 2024-04-25T04:29:54.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:54 smithi112 ceph-mon[50007]: osdmap e154: 8 total, 8 up, 8 in 2024-04-25T04:29:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:54 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 253/1602 objects degraded (15.793%), 35 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:54 smithi112 ceph-mon[50007]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:29:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:54 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3603056724' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:29:54.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:54 smithi112 ceph-mon[50007]: pgmap v300: 97 pgs: 1 activating, 3 remapped+peering, 4 active+undersized, 32 active+undersized+degraded, 57 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 526 op/s; 216/1596 objects degraded (13.534%) 2024-04-25T04:29:54.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285255 9.88 MB/sec execute 578 sec latency 2139.428 ms 2024-04-25T04:29:55.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:54 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[62938]: 2024-04-25T04:29:54.884+0000 7f6f1bde4700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:29:55.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: osdmap e155: 8 total, 8 up, 8 in 2024-04-25T04:29:55.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-25T04:29:55.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:29:55.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: osdmap e156: 8 total, 8 up, 8 in 2024-04-25T04:29:55.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: Standby daemon mds.foofs.smithi112.xminls assigned to filesystem foofs as rank 0 2024-04-25T04:29:55.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-25T04:29:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T04:29:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-25T04:29:55.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-mon[62942]: fsmap foofs:1/1 {0=foofs.smithi112.xminls=up:replay} 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: osdmap e155: 8 total, 8 up, 8 in 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: osdmap e156: 8 total, 8 up, 8 in 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: Standby daemon mds.foofs.smithi112.xminls assigned to filesystem foofs as rank 0 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-25T04:29:55.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:55 smithi112 ceph-mon[50007]: fsmap foofs:1/1 {0=foofs.smithi112.xminls=up:replay} 2024-04-25T04:29:55.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285264 9.86 MB/sec execute 579 sec latency 991.234 ms 2024-04-25T04:29:56.354 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:29:55 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:29:55.926+0000 7f53cb144700 -1 bdev(0x5580f4182000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 5 2024-04-25T04:29:56.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285264 9.84 MB/sec execute 580 sec latency 1991.293 ms 2024-04-25T04:29:57.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:56 smithi112 ceph-mon[50007]: osdmap e157: 8 total, 8 up, 8 in 2024-04-25T04:29:57.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:56 smithi112 ceph-mon[50007]: pgmap v304: 97 pgs: 1 peering, 1 activating, 3 remapped+peering, 2 active+undersized, 27 active+undersized+degraded, 63 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 744 KiB/s wr, 49 op/s; 183/1554 objects degraded (11.776%); 4.0 KiB/s, 6 objects/s recovering 2024-04-25T04:29:57.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:56 smithi078 ceph-mon[62942]: osdmap e157: 8 total, 8 up, 8 in 2024-04-25T04:29:57.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:56 smithi078 ceph-mon[62942]: pgmap v304: 97 pgs: 1 peering, 1 activating, 3 remapped+peering, 2 active+undersized, 27 active+undersized+degraded, 63 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 744 KiB/s wr, 49 op/s; 183/1554 objects degraded (11.776%); 4.0 KiB/s, 6 objects/s recovering 2024-04-25T04:29:57.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285264 9.83 MB/sec execute 581 sec latency 2991.374 ms 2024-04-25T04:29:58.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:57 smithi112 ceph-mon[50007]: reconnect by client.24325 172.21.15.112:0/3701026968 after 0.000999974 2024-04-25T04:29:58.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:57 smithi112 ceph-mon[50007]: reconnect by client.14560 172.21.15.78:0/4069364465 after 0.000999974 2024-04-25T04:29:58.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:57 smithi112 ceph-mon[50007]: osdmap e158: 8 total, 8 up, 8 in 2024-04-25T04:29:58.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:57 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:reconnect 2024-04-25T04:29:58.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:57 smithi112 ceph-mon[50007]: fsmap foofs:1/1 {0=foofs.smithi112.xminls=up:reconnect} 2024-04-25T04:29:58.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:57 smithi078 ceph-mon[62942]: reconnect by client.24325 172.21.15.112:0/3701026968 after 0.000999974 2024-04-25T04:29:58.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:57 smithi078 ceph-mon[62942]: reconnect by client.14560 172.21.15.78:0/4069364465 after 0.000999974 2024-04-25T04:29:58.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:57 smithi078 ceph-mon[62942]: osdmap e158: 8 total, 8 up, 8 in 2024-04-25T04:29:58.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:57 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:reconnect 2024-04-25T04:29:58.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:57 smithi078 ceph-mon[62942]: fsmap foofs:1/1 {0=foofs.smithi112.xminls=up:reconnect} 2024-04-25T04:29:58.872 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285264 9.81 MB/sec execute 582 sec latency 3991.398 ms 2024-04-25T04:29:59.141 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:58 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:rejoin 2024-04-25T04:29:59.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:58 smithi078 ceph-mon[62942]: fsmap foofs:1/1 {0=foofs.smithi112.xminls=up:rejoin} 2024-04-25T04:29:59.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:58 smithi078 ceph-mon[62942]: daemon mds.foofs.smithi112.xminls is now active in filesystem foofs as rank 0 2024-04-25T04:29:59.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:58 smithi078 ceph-mon[62942]: pgmap v306: 97 pgs: 2 peering, 1 activating, 2 remapped+peering, 92 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 210 op/s; 4.1 MiB/s, 58 objects/s recovering 2024-04-25T04:29:59.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:58 smithi078 ceph-mon[62942]: Health check update: Degraded data redundancy: 183/1554 objects degraded (11.776%), 27 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:59.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:58 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:rejoin 2024-04-25T04:29:59.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:58 smithi112 ceph-mon[50007]: fsmap foofs:1/1 {0=foofs.smithi112.xminls=up:rejoin} 2024-04-25T04:29:59.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:58 smithi112 ceph-mon[50007]: daemon mds.foofs.smithi112.xminls is now active in filesystem foofs as rank 0 2024-04-25T04:29:59.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:58 smithi112 ceph-mon[50007]: pgmap v306: 97 pgs: 2 peering, 1 activating, 2 remapped+peering, 92 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 210 op/s; 4.1 MiB/s, 58 objects/s recovering 2024-04-25T04:29:59.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:58 smithi112 ceph-mon[50007]: Health check update: Degraded data redundancy: 183/1554 objects degraded (11.776%), 27 pgs degraded (PG_DEGRADED) 2024-04-25T04:29:59.876 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285561 9.81 MB/sec execute 583 sec latency 4049.282 ms 2024-04-25T04:29:59.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:59 smithi078 ceph-mon[62942]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 183/1554 objects degraded (11.776%), 27 pgs degraded) 2024-04-25T04:29:59.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:59 smithi078 ceph-mon[62942]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-25T04:29:59.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:59 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:active 2024-04-25T04:29:59.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:29:59 smithi078 ceph-mon[62942]: fsmap foofs:1 {0=foofs.smithi112.xminls=up:active} 2024-04-25T04:30:00.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:59 smithi112 ceph-mon[50007]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 183/1554 objects degraded (11.776%), 27 pgs degraded) 2024-04-25T04:30:00.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:59 smithi112 ceph-mon[50007]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-25T04:30:00.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:59 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.112:6832/1324789647,v1:172.21.15.112:6833/1324789647] up:active 2024-04-25T04:30:00.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:29:59 smithi112 ceph-mon[50007]: fsmap foofs:1 {0=foofs.smithi112.xminls=up:active} 2024-04-25T04:30:00.876 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285882 9.82 MB/sec execute 584 sec latency 45.330 ms 2024-04-25T04:30:00.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:00 smithi078 ceph-mon[62942]: Health detail: HEALTH_WARN insufficient standby MDS daemons available 2024-04-25T04:30:00.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:00 smithi078 ceph-mon[62942]: [WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-04-25T04:30:00.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:00 smithi078 ceph-mon[62942]: have 0; want 1 more 2024-04-25T04:30:00.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:00 smithi078 ceph-mon[62942]: pgmap v307: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 155 op/s; 2.8 MiB/s, 46 objects/s recovering 2024-04-25T04:30:01.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:00 smithi112 ceph-mon[50007]: Health detail: HEALTH_WARN insufficient standby MDS daemons available 2024-04-25T04:30:01.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:00 smithi112 ceph-mon[50007]: [WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-04-25T04:30:01.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:00 smithi112 ceph-mon[50007]: have 0; want 1 more 2024-04-25T04:30:01.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:00 smithi112 ceph-mon[50007]: pgmap v307: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 155 op/s; 2.8 MiB/s, 46 objects/s recovering 2024-04-25T04:30:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 286248 9.82 MB/sec execute 585 sec latency 55.383 ms 2024-04-25T04:30:02.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 286637 9.82 MB/sec execute 586 sec latency 37.200 ms 2024-04-25T04:30:03.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:03 smithi078 ceph-mon[62942]: pgmap v308: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.5 MiB/s wr, 278 op/s; 2.5 MiB/s, 42 objects/s recovering 2024-04-25T04:30:03.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:03 smithi112 ceph-mon[50007]: pgmap v308: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.5 MiB/s wr, 278 op/s; 2.5 MiB/s, 42 objects/s recovering 2024-04-25T04:30:03.878 INFO:teuthology.orchestra.run.smithi078.stdout: 5 286950 9.82 MB/sec execute 587 sec latency 45.979 ms 2024-04-25T04:30:04.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 287222 9.82 MB/sec execute 588 sec latency 110.088 ms 2024-04-25T04:30:05.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:05 smithi078 ceph-mon[62942]: pgmap v309: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 2.4 MiB/s wr, 253 op/s; 2.2 MiB/s, 34 objects/s recovering 2024-04-25T04:30:05.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:05 smithi112 ceph-mon[50007]: pgmap v309: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 MiB/s rd, 2.4 MiB/s wr, 253 op/s; 2.2 MiB/s, 34 objects/s recovering 2024-04-25T04:30:05.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 287581 9.81 MB/sec execute 589 sec latency 47.403 ms 2024-04-25T04:30:06.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 287899 9.81 MB/sec execute 590 sec latency 47.195 ms 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: pgmap v310: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.3 MiB/s wr, 224 op/s; 1.8 MiB/s, 28 objects/s recovering 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:boot 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: fsmap foofs:1 {0=foofs.smithi112.xminls=up:active} 1 up:standby 2024-04-25T04:30:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:07 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:30:07.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 288216 9.81 MB/sec execute 591 sec latency 41.748 ms 2024-04-25T04:30:07.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: pgmap v310: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.3 MiB/s wr, 224 op/s; 1.8 MiB/s, 28 objects/s recovering 2024-04-25T04:30:07.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:07.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:07.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:07.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-25T04:30:07.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:30:07.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:boot 2024-04-25T04:30:07.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: fsmap foofs:1 {0=foofs.smithi112.xminls=up:active} 1 up:standby 2024-04-25T04:30:07.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:07 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:30:08.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 288506 9.80 MB/sec execute 592 sec latency 53.431 ms 2024-04-25T04:30:09.344 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:09.344 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:09.344 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:09.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:09.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:09.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:09.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 288781 9.79 MB/sec execute 593 sec latency 62.185 ms 2024-04-25T04:30:10.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:10 smithi078 ceph-mon[62942]: pgmap v311: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.3 MiB/s wr, 343 op/s; 168 KiB/s, 6 objects/s recovering 2024-04-25T04:30:10.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:10 smithi112 ceph-mon[50007]: pgmap v311: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.3 MiB/s wr, 343 op/s; 168 KiB/s, 6 objects/s recovering 2024-04-25T04:30:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 289066 9.79 MB/sec execute 594 sec latency 48.078 ms 2024-04-25T04:30:11.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 289237 9.78 MB/sec execute 595 sec latency 61.721 ms 2024-04-25T04:30:12.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:12 smithi078 ceph-mon[62942]: pgmap v312: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 344 op/s; 158 KiB/s, 6 objects/s recovering 2024-04-25T04:30:12.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:12 smithi112 ceph-mon[50007]: pgmap v312: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 344 op/s; 158 KiB/s, 6 objects/s recovering 2024-04-25T04:30:12.878 INFO:teuthology.orchestra.run.smithi078.stdout: 5 289492 9.79 MB/sec execute 596 sec latency 71.564 ms 2024-04-25T04:30:13.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 289744 9.78 MB/sec execute 597 sec latency 261.031 ms 2024-04-25T04:30:14.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:14 smithi078 ceph-mon[62942]: pgmap v313: 97 pgs: 97 active+clean; 338 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.7 MiB/s wr, 495 op/s; 90 KiB/s, 2 objects/s recovering 2024-04-25T04:30:14.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:14 smithi112 ceph-mon[50007]: pgmap v313: 97 pgs: 97 active+clean; 338 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.7 MiB/s wr, 495 op/s; 90 KiB/s, 2 objects/s recovering 2024-04-25T04:30:14.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 290065 9.78 MB/sec execute 598 sec latency 60.505 ms 2024-04-25T04:30:15.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 290306 9.77 MB/sec execute 599 sec latency 78.747 ms 2024-04-25T04:30:16.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:16 smithi078 ceph-mon[62942]: pgmap v314: 97 pgs: 97 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-04-25T04:30:16.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:16 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:16.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:16 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:16.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:16 smithi112 ceph-mon[50007]: pgmap v314: 97 pgs: 97 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 4.0 MiB/s wr, 410 op/s 2024-04-25T04:30:16.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:16 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:16.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:16 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:16.877 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 600 sec 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 600 sec 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 187686 4.539 2126.899 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: Close 137764 0.130 10.042 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: Rename 7954 9.624 2139.027 2024-04-25T04:30:17.281 INFO:teuthology.orchestra.run.smithi078.stdout: Unlink 37980 7.192 2100.898 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: Qpathinfo 170241 1.137 90.184 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 29680 0.006 0.823 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: Qfsinfo 31282 0.587 18.315 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: Sfileinfo 15257 13.525 4049.274 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: Find 65852 0.920 37.093 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 92760 13.958 4039.213 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: ReadX 294981 0.010 6.713 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: LockX 616 0.438 10.410 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: UnlockX 616 0.399 10.828 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: Flush 13148 0.002 0.048 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:30:17.282 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 9.77463 MB/sec 5 clients 5 procs max_latency=4049.282 ms 2024-04-25T04:30:17.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-04-25T04:30:17.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-04-25T04:30:17.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-04-25T04:30:17.490 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-25T04:30:17.490 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:30:17.490 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-04-25T04:30:17.491 INFO:teuthology.orchestra.run.smithi078.stdout:4 of 5 processes prepared for launch 0 sec 2024-04-25T04:30:17.591 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-25T04:30:17.592 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-04-25T04:30:17.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:17 smithi078 ceph-mon[62942]: pgmap v315: 97 pgs: 97 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 4.2 MiB/s wr, 457 op/s 2024-04-25T04:30:17.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:17.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:17 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:18.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:17 smithi112 ceph-mon[50007]: pgmap v315: 97 pgs: 97 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 4.2 MiB/s wr, 457 op/s 2024-04-25T04:30:18.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:18.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:17 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:19.592 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256 15.61 MB/sec execute 1 sec latency 119.695 ms 2024-04-25T04:30:20.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:19 smithi078 ceph-mon[62942]: pgmap v316: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-25T04:30:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:19 smithi112 ceph-mon[50007]: pgmap v316: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-04-25T04:30:20.591 INFO:teuthology.orchestra.run.smithi078.stdout: 5 347 14.70 MB/sec execute 2 sec latency 114.025 ms 2024-04-25T04:30:20.819 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:20 smithi078 ceph-mon[62942]: pgmap v317: 97 pgs: 97 active+clean; 341 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 5.0 MiB/s wr, 357 op/s 2024-04-25T04:30:21.051 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:20 smithi112 ceph-mon[50007]: pgmap v317: 97 pgs: 97 active+clean; 341 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 5.0 MiB/s wr, 357 op/s 2024-04-25T04:30:21.591 INFO:teuthology.orchestra.run.smithi078.stdout: 5 447 15.26 MB/sec execute 3 sec latency 106.652 ms 2024-04-25T04:30:22.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: Detected new or changed devices on smithi078 2024-04-25T04:30:22.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:22.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:30:22.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi112.xminls"]}]: dispatch 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: Upgrade: It appears safe to stop mds.foofs.smithi112.xminls 2024-04-25T04:30:22.490 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:22 smithi078 ceph-mon[62942]: pgmap v318: 97 pgs: 97 active+clean; 308 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 459 op/s 2024-04-25T04:30:22.494 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: Detected new or changed devices on smithi078 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi112.xminls"]}]: dispatch 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: Upgrade: It appears safe to stop mds.foofs.smithi112.xminls 2024-04-25T04:30:22.495 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:22 smithi112 ceph-mon[50007]: pgmap v318: 97 pgs: 97 active+clean; 308 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 459 op/s 2024-04-25T04:30:22.591 INFO:teuthology.orchestra.run.smithi078.stdout: 5 546 15.64 MB/sec execute 4 sec latency 127.789 ms 2024-04-25T04:30:23.546 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:23.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:23.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:30:23.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:23.591 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 5 sec 2024-04-25T04:30:23.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:23.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:23.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi112.xminls", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-25T04:30:23.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:24.360 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:30:24.396 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 6 sec 2024-04-25T04:30:24.396 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:30:24.396 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 370 30.770 114.021 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Close 357 14.972 127.779 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Rename 6 31.157 50.377 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Unlink 13 6.765 11.649 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Qpathinfo 45 0.870 20.532 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 314 20.635 98.073 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Qfsinfo 53 0.353 0.615 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Sfileinfo 30 17.151 92.816 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: Find 44 0.765 3.448 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 1499 0.026 0.074 2024-04-25T04:30:24.397 INFO:teuthology.orchestra.run.smithi078.stdout: ReadX 167 0.013 0.239 2024-04-25T04:30:24.398 INFO:teuthology.orchestra.run.smithi078.stdout: Flush 60 16.604 27.252 2024-04-25T04:30:24.398 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:30:24.398 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 15.6351 MB/sec 5 clients 5 procs max_latency=127.789 ms 2024-04-25T04:30:24.399 DEBUG:teuthology.parallel:result is None 2024-04-25T04:30:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:24 smithi078 ceph-mon[62942]: Upgrade: Updating mds.foofs.smithi112.xminls 2024-04-25T04:30:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:24 smithi078 ceph-mon[62942]: Deploying daemon mds.foofs.smithi112.xminls on smithi112 2024-04-25T04:30:24.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:24 smithi078 ceph-mon[62942]: pgmap v319: 97 pgs: 97 active+clean; 297 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.1 MiB/s wr, 312 op/s 2024-04-25T04:30:24.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:24 smithi112 ceph-mon[50007]: Upgrade: Updating mds.foofs.smithi112.xminls 2024-04-25T04:30:24.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:24 smithi112 ceph-mon[50007]: Deploying daemon mds.foofs.smithi112.xminls on smithi112 2024-04-25T04:30:24.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:24 smithi112 ceph-mon[50007]: pgmap v319: 97 pgs: 97 active+clean; 297 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.1 MiB/s wr, 312 op/s 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 9s ago 18m 24.8M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 9s ago 18m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (6m) 42s ago 18m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (9m) 9s ago 18m 55.3M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (19s) 9s ago 12m 10.3M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 479f42f132ed 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (12m) 42s ago 12m 15.5M - 17.2.0 e1d6a67b021e 27d61193b021 2024-04-25T04:30:25.066 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (9m) 9s ago 20m 514M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (8m) 42s ago 18m 453M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 9s ago 20m 59.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (7m) 42s ago 17m 44.0M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (12m) 9s ago 12m 289M - 4.0 e1d6a67b021e 6451f5d61c49 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (12m) 42s ago 12m 88.0M - 4.0 e1d6a67b021e 73293a1c2b13 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 9s ago 18m 20.2M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (9m) 42s ago 17m 18.7M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 9s ago 17m 444M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 9s ago 16m 402M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 9s ago 16m 349M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 9s ago 15m 336M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (2m) 42s ago 15m 323M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (2m) 42s ago 14m 293M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:30:25.067 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (83s) 42s ago 14m 283M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:30:25.068 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (46s) 42s ago 14m 13.6M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:30:25.068 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 9s ago 17m 75.5M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:30:25.477 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:30:25.477 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:30:25.477 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:30:25.477 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:25.477 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 1 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:30:25.478 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-25T04:30:25.479 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 13 2024-04-25T04:30:25.479 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:30:25.479 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:30:25.584 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:25 smithi112 ceph-mon[50007]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:25.585 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:25 smithi112 ceph-mon[50007]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:25.585 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:25 smithi112 ceph-mon[50007]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:25.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-mon[62942]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:25.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-mon[62942]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:25.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-mon[62942]: from='client.34500 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:25.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[62938]: 2024-04-25T04:30:25.550+0000 7f6f1bde4700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:30:25.855 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:30:25.579+0000 7f53cc146700 -1 bdev(0x5580f4182000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 10 2024-04-25T04:30:25.855 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:30:25.579+0000 7f53cf94d700 -1 bdev(0x5580f4182000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 15 2024-04-25T04:30:25.855 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:30:25.579+0000 7f53cb144700 -1 bdev(0x5580f4182000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 16 2024-04-25T04:30:25.855 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:30:25 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:30:25.579+0000 7f53cb945700 -1 bdev(0x5580f4182000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 14 2024-04-25T04:30:25.861 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:30:25.861 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "15/23 daemons upgraded", 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:30:25.862 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:30:25.899 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:30:25 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:30:25.579+0000 7f49de44b700 -1 bdev(0x55c95bf3a000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-04-25T04:30:26.305 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 filesystem is degraded; insufficient standby MDS daemons available 2024-04-25T04:30:26.305 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-04-25T04:30:26.305 INFO:teuthology.orchestra.run.smithi078.stdout: fs foofs is degraded 2024-04-25T04:30:26.305 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-04-25T04:30:26.305 INFO:teuthology.orchestra.run.smithi078.stdout: have 0; want 1 more 2024-04-25T04:30:26.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/589069628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:26.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-25T04:30:26.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:30:26.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: osdmap e159: 8 total, 8 up, 8 in 2024-04-25T04:30:26.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: Standby daemon mds.foofs.smithi078.nlrtle assigned to filesystem foofs as rank 0 2024-04-25T04:30:26.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-25T04:30:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T04:30:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-25T04:30:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: fsmap foofs:1/1 {0=foofs.smithi078.nlrtle=up:replay} 2024-04-25T04:30:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: pgmap v321: 97 pgs: 97 active+clean; 306 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 306 op/s 2024-04-25T04:30:26.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:26 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2406979990' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:30:26.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/589069628' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:26.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: osdmap e159: 8 total, 8 up, 8 in 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: Standby daemon mds.foofs.smithi078.nlrtle assigned to filesystem foofs as rank 0 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: fsmap foofs:1/1 {0=foofs.smithi078.nlrtle=up:replay} 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: pgmap v321: 97 pgs: 97 active+clean; 306 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 306 op/s 2024-04-25T04:30:26.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:26 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2406979990' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:30:28.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:28 smithi078 ceph-mon[62942]: reconnect by client.14560 172.21.15.78:0/4069364465 after 0 2024-04-25T04:30:28.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:28 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:reconnect 2024-04-25T04:30:28.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:28 smithi078 ceph-mon[62942]: fsmap foofs:1/1 {0=foofs.smithi078.nlrtle=up:reconnect} 2024-04-25T04:30:28.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:28 smithi078 ceph-mon[62942]: reconnect by client.24325 172.21.15.112:0/3701026968 after 0.0539992 2024-04-25T04:30:28.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:28 smithi078 ceph-mon[62942]: pgmap v322: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 12 MiB/s wr, 321 op/s 2024-04-25T04:30:28.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:28 smithi112 ceph-mon[50007]: reconnect by client.14560 172.21.15.78:0/4069364465 after 0 2024-04-25T04:30:28.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:28 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:reconnect 2024-04-25T04:30:28.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:28 smithi112 ceph-mon[50007]: fsmap foofs:1/1 {0=foofs.smithi078.nlrtle=up:reconnect} 2024-04-25T04:30:28.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:28 smithi112 ceph-mon[50007]: reconnect by client.24325 172.21.15.112:0/3701026968 after 0.0539992 2024-04-25T04:30:28.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:28 smithi112 ceph-mon[50007]: pgmap v322: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 12 MiB/s wr, 321 op/s 2024-04-25T04:30:29.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:29 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:rejoin 2024-04-25T04:30:29.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:29 smithi078 ceph-mon[62942]: fsmap foofs:1/1 {0=foofs.smithi078.nlrtle=up:rejoin} 2024-04-25T04:30:29.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:29 smithi078 ceph-mon[62942]: daemon mds.foofs.smithi078.nlrtle is now active in filesystem foofs as rank 0 2024-04-25T04:30:29.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:29 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:30:29.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:29 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:rejoin 2024-04-25T04:30:29.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:29 smithi112 ceph-mon[50007]: fsmap foofs:1/1 {0=foofs.smithi078.nlrtle=up:rejoin} 2024-04-25T04:30:29.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:29 smithi112 ceph-mon[50007]: daemon mds.foofs.smithi078.nlrtle is now active in filesystem foofs as rank 0 2024-04-25T04:30:29.900 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:29 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:30:30.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:30 smithi078 ceph-mon[62942]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-25T04:30:30.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:30 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:active 2024-04-25T04:30:30.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:30 smithi078 ceph-mon[62942]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 2024-04-25T04:30:30.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:30 smithi078 ceph-mon[62942]: pgmap v323: 97 pgs: 97 active+clean; 304 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 9.6 MiB/s wr, 262 op/s 2024-04-25T04:30:30.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:30 smithi112 ceph-mon[50007]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-25T04:30:30.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:30 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136] up:active 2024-04-25T04:30:30.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:30 smithi112 ceph-mon[50007]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 2024-04-25T04:30:30.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:30 smithi112 ceph-mon[50007]: pgmap v323: 97 pgs: 97 active+clean; 304 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 9.6 MiB/s wr, 262 op/s 2024-04-25T04:30:32.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:32 smithi078 ceph-mon[62942]: pgmap v324: 97 pgs: 97 active+clean; 240 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.5 MiB/s wr, 139 op/s 2024-04-25T04:30:32.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:32 smithi112 ceph-mon[50007]: pgmap v324: 97 pgs: 97 active+clean; 240 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.5 MiB/s wr, 139 op/s 2024-04-25T04:30:34.447 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:34 smithi112 ceph-mon[50007]: pgmap v325: 97 pgs: 97 active+clean; 225 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.7 MiB/s wr, 130 op/s 2024-04-25T04:30:34.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:34 smithi078 ceph-mon[62942]: pgmap v325: 97 pgs: 97 active+clean; 225 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 3.7 MiB/s wr, 130 op/s 2024-04-25T04:30:36.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:35 smithi078 ceph-mon[62942]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-25T04:30:36.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:35 smithi078 ceph-mon[62942]: Cluster is now healthy 2024-04-25T04:30:36.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:35 smithi078 ceph-mon[62942]: mds.? [v2:172.21.15.112:6832/1971356950,v1:172.21.15.112:6833/1971356950] up:boot 2024-04-25T04:30:36.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:35 smithi078 ceph-mon[62942]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:30:36.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:35 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:30:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:35 smithi112 ceph-mon[50007]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-25T04:30:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:35 smithi112 ceph-mon[50007]: Cluster is now healthy 2024-04-25T04:30:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:35 smithi112 ceph-mon[50007]: mds.? [v2:172.21.15.112:6832/1971356950,v1:172.21.15.112:6833/1971356950] up:boot 2024-04-25T04:30:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:35 smithi112 ceph-mon[50007]: fsmap foofs:1 {0=foofs.smithi078.nlrtle=up:active} 1 up:standby 2024-04-25T04:30:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:35 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:30:36.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:36 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:36.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:36 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:36.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:36 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:37.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:36 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:37.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:36 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:37.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:36 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:38.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:37 smithi078 ceph-mon[62942]: pgmap v326: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.3 MiB/s wr, 112 op/s 2024-04-25T04:30:38.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:37 smithi112 ceph-mon[50007]: pgmap v326: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.3 MiB/s wr, 112 op/s 2024-04-25T04:30:39.096 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:39.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:40.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:39 smithi112 ceph-mon[50007]: pgmap v327: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 103 op/s 2024-04-25T04:30:40.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:40.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:40.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:39 smithi078 ceph-mon[62942]: pgmap v327: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 2.1 MiB/s wr, 103 op/s 2024-04-25T04:30:40.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:40.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:41.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:40 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:41.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:40 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:41.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:40 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:41.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:40 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:42.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:41 smithi112 ceph-mon[50007]: pgmap v328: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 25 KiB/s wr, 33 op/s 2024-04-25T04:30:42.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:41 smithi078 ceph-mon[62942]: pgmap v328: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 25 KiB/s wr, 33 op/s 2024-04-25T04:30:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: pgmap v329: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 26 KiB/s wr, 27 op/s 2024-04-25T04:30:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:30:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.nlrtle"}]': finished 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.xminls"}]': finished 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: pgmap v329: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 26 KiB/s wr, 27 op/s 2024-04-25T04:30:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:30:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.nlrtle"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.nlrtle"}]': finished 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.xminls"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi112.xminls"}]': finished 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:44.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:44.402 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:45.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:44 smithi078 ceph-mon[62942]: Detected new or changed devices on smithi112 2024-04-25T04:30:45.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:44 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all mds 2024-04-25T04:30:45.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:44 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all rgw 2024-04-25T04:30:45.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:44 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all rbd-mirror 2024-04-25T04:30:45.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:44 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all ceph-exporter 2024-04-25T04:30:45.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:44 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all iscsi 2024-04-25T04:30:45.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:44 smithi112 ceph-mon[50007]: Detected new or changed devices on smithi112 2024-04-25T04:30:45.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:44 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all mds 2024-04-25T04:30:45.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:44 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all rgw 2024-04-25T04:30:45.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:44 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all rbd-mirror 2024-04-25T04:30:45.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:44 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all ceph-exporter 2024-04-25T04:30:45.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:44 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all iscsi 2024-04-25T04:30:46.282 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: pgmap v330: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 23 KiB/s wr, 7 op/s 2024-04-25T04:30:46.282 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:30:46.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:45 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:46.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: pgmap v330: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 23 KiB/s wr, 7 op/s 2024-04-25T04:30:46.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:46.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:30:46.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:30:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:30:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:30:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:30:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.edyrsx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:30:46.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:45 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:47.191 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:46 smithi078 ceph-mon[62942]: Upgrade: Updating nfs.foo.0.0.smithi078.edyrsx (1/2) 2024-04-25T04:30:47.191 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:46 smithi078 ceph-mon[62942]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx 2024-04-25T04:30:47.191 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:46 smithi078 ceph-mon[62942]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-25T04:30:47.192 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:46 smithi078 ceph-mon[62942]: Rados config object exists: conf-nfs.foo 2024-04-25T04:30:47.192 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:46 smithi078 ceph-mon[62942]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx-rgw 2024-04-25T04:30:47.192 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:46 smithi078 ceph-mon[62942]: Bind address in nfs.foo.0.0.smithi078.edyrsx's ganesha conf is defaulting to empty 2024-04-25T04:30:47.192 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:46 smithi078 ceph-mon[62942]: Deploying daemon nfs.foo.0.0.smithi078.edyrsx on smithi078 2024-04-25T04:30:47.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:46 smithi112 ceph-mon[50007]: Upgrade: Updating nfs.foo.0.0.smithi078.edyrsx (1/2) 2024-04-25T04:30:47.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:46 smithi112 ceph-mon[50007]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx 2024-04-25T04:30:47.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:46 smithi112 ceph-mon[50007]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-25T04:30:47.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:46 smithi112 ceph-mon[50007]: Rados config object exists: conf-nfs.foo 2024-04-25T04:30:47.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:46 smithi112 ceph-mon[50007]: Creating key for client.nfs.foo.0.0.smithi078.edyrsx-rgw 2024-04-25T04:30:47.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:46 smithi112 ceph-mon[50007]: Bind address in nfs.foo.0.0.smithi078.edyrsx's ganesha conf is defaulting to empty 2024-04-25T04:30:47.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:46 smithi112 ceph-mon[50007]: Deploying daemon nfs.foo.0.0.smithi078.edyrsx on smithi078 2024-04-25T04:30:48.203 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:47 smithi078 ceph-mon[62942]: pgmap v331: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 23 KiB/s wr, 4 op/s 2024-04-25T04:30:48.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:47 smithi112 ceph-mon[50007]: pgmap v331: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 23 KiB/s wr, 4 op/s 2024-04-25T04:30:50.346 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:49 smithi112 ceph-mon[50007]: pgmap v332: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 24 KiB/s wr, 4 op/s 2024-04-25T04:30:50.346 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:49 smithi112 ceph-mon[50007]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/4069364465"}]: dispatch 2024-04-25T04:30:50.346 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:50.347 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:49 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:50.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:49 smithi078 ceph-mon[62942]: pgmap v332: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 24 KiB/s wr, 4 op/s 2024-04-25T04:30:50.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:49 smithi078 ceph-mon[62942]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/4069364465"}]: dispatch 2024-04-25T04:30:50.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:50.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:49 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:51.285 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:50 smithi112 ceph-mon[50007]: Evicting (and blocklisting) client session 14560 (172.21.15.78:0/4069364465) 2024-04-25T04:30:51.285 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:50 smithi112 ceph-mon[50007]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/4069364465"}]': finished 2024-04-25T04:30:51.285 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:50 smithi112 ceph-mon[50007]: osdmap e160: 8 total, 8 up, 8 in 2024-04-25T04:30:51.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:50 smithi078 ceph-mon[62942]: Evicting (and blocklisting) client session 14560 (172.21.15.78:0/4069364465) 2024-04-25T04:30:51.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:50 smithi078 ceph-mon[62942]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/4069364465"}]': finished 2024-04-25T04:30:51.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:50 smithi078 ceph-mon[62942]: osdmap e160: 8 total, 8 up, 8 in 2024-04-25T04:30:52.221 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: pgmap v334: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 1.1 KiB/s wr, 0 op/s 2024-04-25T04:30:52.221 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:30:52.222 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:51 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:52.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: pgmap v334: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 1.1 KiB/s wr, 0 op/s 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi112.ouafxs-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-25T04:30:52.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:51 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:30:53.213 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:52 smithi112 ceph-mon[50007]: Upgrade: Updating nfs.foo.1.0.smithi112.ouafxs (2/2) 2024-04-25T04:30:53.214 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:52 smithi112 ceph-mon[50007]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs 2024-04-25T04:30:53.214 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:52 smithi112 ceph-mon[50007]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-25T04:30:53.214 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:52 smithi112 ceph-mon[50007]: Rados config object exists: conf-nfs.foo 2024-04-25T04:30:53.214 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:52 smithi112 ceph-mon[50007]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs-rgw 2024-04-25T04:30:53.214 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:52 smithi112 ceph-mon[50007]: Bind address in nfs.foo.1.0.smithi112.ouafxs's ganesha conf is defaulting to empty 2024-04-25T04:30:53.214 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:52 smithi112 ceph-mon[50007]: Deploying daemon nfs.foo.1.0.smithi112.ouafxs on smithi112 2024-04-25T04:30:53.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:52 smithi078 ceph-mon[62942]: Upgrade: Updating nfs.foo.1.0.smithi112.ouafxs (2/2) 2024-04-25T04:30:53.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:52 smithi078 ceph-mon[62942]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs 2024-04-25T04:30:53.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:52 smithi078 ceph-mon[62942]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-25T04:30:53.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:52 smithi078 ceph-mon[62942]: Rados config object exists: conf-nfs.foo 2024-04-25T04:30:53.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:52 smithi078 ceph-mon[62942]: Creating key for client.nfs.foo.1.0.smithi112.ouafxs-rgw 2024-04-25T04:30:53.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:52 smithi078 ceph-mon[62942]: Bind address in nfs.foo.1.0.smithi112.ouafxs's ganesha conf is defaulting to empty 2024-04-25T04:30:53.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:52 smithi078 ceph-mon[62942]: Deploying daemon nfs.foo.1.0.smithi112.ouafxs on smithi112 2024-04-25T04:30:54.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:53 smithi112 ceph-mon[50007]: pgmap v335: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-25T04:30:54.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:54.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:53 smithi078 ceph-mon[62942]: pgmap v335: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-25T04:30:54.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:30:56.230 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:55 smithi112 ceph-mon[50007]: pgmap v336: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-25T04:30:56.230 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:55 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:56.230 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:55 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:56.230 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:55 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:56.302 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:55 smithi078 ceph-mon[62942]: pgmap v336: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-25T04:30:56.302 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:55 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:56.302 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:55 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:56.303 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:55 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:30:56.699 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:30:57.398 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 41s ago 19m 24.8M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 41s ago 19m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (6m) 18s ago 18m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (10m) 41s ago 19m 55.3M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (51s) 41s ago 13m 10.3M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 479f42f132ed 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (22s) 18s ago 13m 12.2M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 3f760ad5208f 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (9m) 41s ago 20m 514M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (8m) 18s ago 18m 453M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 41s ago 20m 59.8M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (8m) 18s ago 18m 46.7M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 starting - - - - 2024-04-25T04:30:57.399 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 starting - - - - 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (10m) 41s ago 18m 20.2M - 1.3.1 1dbe0e931976 4da99e789f02 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (9m) 18s ago 18m 16.3M - 1.3.1 1dbe0e931976 181817afba68 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 41s ago 17m 444M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 41s ago 17m 402M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 41s ago 16m 349M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 41s ago 16m 336M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (3m) 18s ago 15m 364M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (2m) 18s ago 15m 339M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (116s) 18s ago 15m 342M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (78s) 18s ago 14m 346M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:30:57.400 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 41s ago 18m 75.5M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:30:57.834 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:30:57.835 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 14 2024-04-25T04:30:57.836 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:30:57.836 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:30:58.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:57 smithi078 ceph-mon[62942]: pgmap v337: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-25T04:30:58.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:57 smithi078 ceph-mon[62942]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:58.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:57 smithi078 ceph-mon[62942]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:58.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:57 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1892792953' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-04-25T04:30:58.185 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:30:58.186 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:30:58.186 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "17/23 daemons upgraded", 2024-04-25T04:30:58.186 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading nfs daemons", 2024-04-25T04:30:58.186 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:30:58.186 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:30:58.249 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:57 smithi112 ceph-mon[50007]: pgmap v337: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-25T04:30:58.249 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:57 smithi112 ceph-mon[50007]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:58.249 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:57 smithi112 ceph-mon[50007]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:58.249 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:57 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1892792953' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:30:58.611 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:30:59.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:59 smithi112 ceph-mon[50007]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:59.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:59 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:59.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:59 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:59.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:59 smithi112 ceph-mon[50007]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:59.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:59 smithi112 ceph-mon[50007]: Evicting (and blocklisting) client session 24325 (172.21.15.112:0/3701026968) 2024-04-25T04:30:59.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:59 smithi112 ceph-mon[50007]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/3701026968"}]: dispatch 2024-04-25T04:30:59.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:30:59 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3484049147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:30:59.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:59 smithi078 ceph-mon[62942]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:59.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:59 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:59.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:59 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:30:59.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:59 smithi078 ceph-mon[62942]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:30:59.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:59 smithi078 ceph-mon[62942]: Evicting (and blocklisting) client session 24325 (172.21.15.112:0/3701026968) 2024-04-25T04:30:59.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:59 smithi078 ceph-mon[62942]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/3701026968"}]: dispatch 2024-04-25T04:30:59.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:30:59 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3484049147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:31:00.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:00 smithi078 ceph-mon[62942]: pgmap v338: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-25T04:31:00.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:00 smithi078 ceph-mon[62942]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/3701026968"}]': finished 2024-04-25T04:31:00.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:00 smithi078 ceph-mon[62942]: osdmap e161: 8 total, 8 up, 8 in 2024-04-25T04:31:00.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:00 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:00.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:00 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:00.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:00 smithi112 ceph-mon[50007]: pgmap v338: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-25T04:31:00.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:00 smithi112 ceph-mon[50007]: from='mds.? [v2:172.21.15.78:6834/2002799136,v1:172.21.15.78:6835/2002799136]' entity='mds.foofs.smithi078.nlrtle' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.112:0/3701026968"}]': finished 2024-04-25T04:31:00.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:00 smithi112 ceph-mon[50007]: osdmap e161: 8 total, 8 up, 8 in 2024-04-25T04:31:00.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:00 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:00.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:00 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:01.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:01 smithi078 ceph-mon[62942]: pgmap v340: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-25T04:31:01.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:01 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:01.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:01 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:01.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:01 smithi112 ceph-mon[50007]: pgmap v340: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-04-25T04:31:01.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:01 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:01.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:01 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:02.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:02.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:02 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:02.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:02.899 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:02 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:03.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:03 smithi078 ceph-mon[62942]: pgmap v341: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-25T04:31:03.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:03 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:03.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:03 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:04.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:03 smithi112 ceph-mon[50007]: pgmap v341: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-25T04:31:04.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:03 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:04.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:03 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:06.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:05 smithi078 ceph-mon[62942]: pgmap v342: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-25T04:31:06.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:05 smithi112 ceph-mon[50007]: pgmap v342: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:07.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all nfs 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.edyrsx"}]: dispatch 2024-04-25T04:31:07.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.edyrsx"}]': finished 2024-04-25T04:31:07.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi112.ouafxs"}]: dispatch 2024-04-25T04:31:07.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi112.ouafxs"}]': finished 2024-04-25T04:31:07.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: Upgrade: Setting container_image for all nvmeof 2024-04-25T04:31:07.106 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:06 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all nfs 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.edyrsx"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.edyrsx"}]': finished 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi112.ouafxs"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi112.ouafxs"}]': finished 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: Upgrade: Setting container_image for all nvmeof 2024-04-25T04:31:07.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:06 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:08.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:07 smithi078 ceph-mon[62942]: pgmap v343: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-25T04:31:08.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:07 smithi112 ceph-mon[50007]: pgmap v343: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-25T04:31:09.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:08 smithi078 ceph-mon[62942]: Upgrade: Updating node-exporter.smithi078 (1/2) 2024-04-25T04:31:09.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:08 smithi078 ceph-mon[62942]: Deploying daemon node-exporter.smithi078 on smithi078 2024-04-25T04:31:09.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:09.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:08 smithi112 ceph-mon[50007]: Upgrade: Updating node-exporter.smithi078 (1/2) 2024-04-25T04:31:09.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:08 smithi112 ceph-mon[50007]: Deploying daemon node-exporter.smithi078 on smithi078 2024-04-25T04:31:09.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:10.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:09 smithi078 ceph-mon[62942]: pgmap v344: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-04-25T04:31:10.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:09 smithi112 ceph-mon[50007]: pgmap v344: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-04-25T04:31:12.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:11 smithi078 ceph-mon[62942]: pgmap v345: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 92 B/s rd, 369 B/s wr, 0 op/s 2024-04-25T04:31:12.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:11 smithi112 ceph-mon[50007]: pgmap v345: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 92 B/s rd, 369 B/s wr, 0 op/s 2024-04-25T04:31:14.225 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:13 smithi112 ceph-mon[50007]: pgmap v346: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-04-25T04:31:14.226 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:13 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:14.226 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:13 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:14.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:13 smithi078 ceph-mon[62942]: pgmap v346: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-04-25T04:31:14.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:13 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:14.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:13 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:16.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:15 smithi112 ceph-mon[50007]: pgmap v347: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:16.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:15 smithi112 ceph-mon[50007]: Upgrade: Updating node-exporter.smithi112 (2/2) 2024-04-25T04:31:16.316 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:15 smithi112 ceph-mon[50007]: Deploying daemon node-exporter.smithi112 on smithi112 2024-04-25T04:31:16.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:15 smithi078 ceph-mon[62942]: pgmap v347: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:16.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:15 smithi078 ceph-mon[62942]: Upgrade: Updating node-exporter.smithi112 (2/2) 2024-04-25T04:31:16.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:15 smithi078 ceph-mon[62942]: Deploying daemon node-exporter.smithi112 on smithi112 2024-04-25T04:31:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:17 smithi078 ceph-mon[62942]: pgmap v348: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:18.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:17 smithi112 ceph-mon[50007]: pgmap v348: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:19 smithi112 ceph-mon[50007]: pgmap v349: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:20.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:19 smithi078 ceph-mon[62942]: pgmap v349: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:21.108 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:21.109 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:21.109 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:21.109 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:20 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:21.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:21.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:21.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:21.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:21.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:20 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:22.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:21 smithi078 ceph-mon[62942]: pgmap v350: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:22.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:21 smithi112 ceph-mon[50007]: pgmap v350: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:24.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:24 smithi078 ceph-mon[62942]: pgmap v351: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:24.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:24.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:24.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:24.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:24 smithi112 ceph-mon[50007]: pgmap v351: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:24.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:24.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:24.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:25.542 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:25 smithi078 ceph-mon[62942]: pgmap v352: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:25.542 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:25 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:25.543 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:25 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:25.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:25 smithi112 ceph-mon[50007]: pgmap v352: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:25.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:25 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:25.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:25 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:27.088 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:26 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:27.088 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:26 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:27.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:26 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:27.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:26 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:27 smithi112 ceph-mon[50007]: pgmap v353: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:28.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:27 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:27 smithi078 ceph-mon[62942]: pgmap v353: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:28.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:27 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:27 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:27 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:27 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:28.986 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:31:29.697 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:31:29.697 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 3s ago 19m 24.9M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:31:29.697 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 3s ago 19m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (7m) 6s ago 19m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (10m) 3s ago 19m 55.7M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (83s) 3s ago 13m 36.2M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 479f42f132ed 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (54s) 6s ago 13m 12.4M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 3f760ad5208f 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (10m) 3s ago 21m 516M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (9m) 6s ago 19m 454M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 3s ago 21m 62.5M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (8m) 6s ago 19m 48.2M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (40s) 3s ago 13m 68.5M - 5.7 3d0c579d3b62 c8ec68b2b94d 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (34s) 6s ago 13m 69.3M - 5.7 3d0c579d3b62 69ae06cbd222 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (16s) 3s ago 19m 7982k - 1.5.0 0da6a335fe13 e730b73491fd 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (10s) 6s ago 18m 3569k - 1.5.0 0da6a335fe13 ed32f4462b95 2024-04-25T04:31:29.698 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 3s ago 18m 457M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 3s ago 17m 408M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 3s ago 17m 355M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 3s ago 16m 362M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (3m) 6s ago 16m 365M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (3m) 6s ago 16m 340M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (2m) 6s ago 15m 343M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (111s) 6s ago 15m 347M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:31:29.699 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (9m) 3s ago 18m 75.5M - 2.33.4 514e6a882f6e 45a73b93f4cb 2024-04-25T04:31:30.191 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:31:30.191 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:31:30.191 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:31:30.191 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 14 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:31:30.192 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:31:30.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:29 smithi078 ceph-mon[62942]: pgmap v354: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:30.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:29 smithi078 ceph-mon[62942]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:30.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:29 smithi112 ceph-mon[50007]: pgmap v354: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:30.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:29 smithi112 ceph-mon[50007]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:30.561 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:31:30.562 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:31:30.562 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:31:30.562 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:31:30.562 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:31:30.562 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:31:30.563 INFO:teuthology.orchestra.run.smithi078.stdout: "nfs", 2024-04-25T04:31:30.563 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-04-25T04:31:30.563 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:31:30.563 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-04-25T04:31:30.563 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:31:30.564 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:31:30.564 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "18/23 daemons upgraded", 2024-04-25T04:31:30.564 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-25T04:31:30.564 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:31:30.564 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:31:31.038 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:31:31.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:31 smithi078 ceph-mon[62942]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:31.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:31 smithi078 ceph-mon[62942]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:31.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:31 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1700911505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:31.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:31 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:31.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:31 smithi112 ceph-mon[50007]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:31.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:31 smithi112 ceph-mon[50007]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:31.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:31 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1700911505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:31.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:31 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:32.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: pgmap v355: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:32.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:32.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:32.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/4118163551' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:32 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: pgmap v355: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/4118163551' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.400 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:32.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:32 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:34.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:34 smithi078 ceph-mon[62942]: pgmap v356: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:34.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:34 smithi078 ceph-mon[62942]: Upgrade: Updating prometheus.smithi078 2024-04-25T04:31:34.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:34 smithi078 ceph-mon[62942]: Deploying daemon prometheus.smithi078 on smithi078 2024-04-25T04:31:34.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:34 smithi112 ceph-mon[50007]: pgmap v356: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:34.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:34 smithi112 ceph-mon[50007]: Upgrade: Updating prometheus.smithi078 2024-04-25T04:31:34.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:34 smithi112 ceph-mon[50007]: Deploying daemon prometheus.smithi078 on smithi078 2024-04-25T04:31:36.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:36 smithi078 ceph-mon[62942]: pgmap v357: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:36.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:36 smithi112 ceph-mon[50007]: pgmap v357: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:38.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:38 smithi078 ceph-mon[62942]: pgmap v358: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:38.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:38 smithi112 ceph-mon[50007]: pgmap v358: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:39.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:39.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:40.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:40 smithi078 ceph-mon[62942]: pgmap v359: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:40.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:40 smithi112 ceph-mon[50007]: pgmap v359: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:42.226 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:42 smithi078 ceph-mon[62942]: pgmap v360: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:42.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:42 smithi112 ceph-mon[50007]: pgmap v360: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:44.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:44 smithi078 ceph-mon[62942]: pgmap v361: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:44.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:44 smithi112 ceph-mon[50007]: pgmap v361: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:46.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:46 smithi078 ceph-mon[62942]: pgmap v362: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:46.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:46.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:46.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:46.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:46 smithi112 ceph-mon[50007]: pgmap v362: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:46.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:46.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:46.401 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:48.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:48 smithi078 ceph-mon[62942]: pgmap v363: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:48.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:48 smithi112 ceph-mon[50007]: pgmap v363: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:50.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:50 smithi078 ceph-mon[62942]: pgmap v364: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:50.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:50 smithi112 ceph-mon[50007]: pgmap v364: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:52.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:52 smithi078 ceph-mon[62942]: pgmap v365: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:52.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:52 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:52.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:52 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:52.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:52 smithi112 ceph-mon[50007]: pgmap v365: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:52.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:52 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:52.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:52 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:53.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:53.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:53.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:53.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:54.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:54 smithi078 ceph-mon[62942]: pgmap v366: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:54.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:54.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:54 smithi112 ceph-mon[50007]: pgmap v366: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:31:54.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:31:56.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:56 smithi078 ceph-mon[62942]: pgmap v367: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:56.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:56 smithi112 ceph-mon[50007]: pgmap v367: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:57.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:57 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:31:57.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:57.650 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:57 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:31:58.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:58 smithi078 ceph-mon[62942]: pgmap v368: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:58.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:58 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:31:58.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:58 smithi112 ceph-mon[50007]: pgmap v368: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:31:58.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:58 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-25T04:31:59.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:59 smithi078 ceph-mon[62942]: Upgrade: Updating alertmanager.smithi078 2024-04-25T04:31:59.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:31:59 smithi078 ceph-mon[62942]: Deploying daemon alertmanager.smithi078 on smithi078 2024-04-25T04:31:59.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:59 smithi112 ceph-mon[50007]: Upgrade: Updating alertmanager.smithi078 2024-04-25T04:31:59.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:31:59 smithi112 ceph-mon[50007]: Deploying daemon alertmanager.smithi078 on smithi078 2024-04-25T04:32:00.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:00 smithi078 ceph-mon[62942]: pgmap v369: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:00.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:00 smithi112 ceph-mon[50007]: pgmap v369: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:01.417 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:32:02.138 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:32:02.138 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (11m) 11s ago 20m 24.9M - 0.23.0 ba2b418f427c dc86c0a88fa5 2024-04-25T04:32:02.138 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 11s ago 20m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:32:02.138 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (8m) 38s ago 19m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (11m) 11s ago 20m 55.7M - 8.3.5 dad864ee21e9 22bca984852e 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (116s) 11s ago 14m 36.4M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 479f42f132ed 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (86s) 38s ago 14m 12.4M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 3f760ad5208f 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (10m) 11s ago 21m 518M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (9m) 38s ago 19m 454M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 11s ago 21m 62.3M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (9m) 38s ago 19m 48.2M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (72s) 11s ago 13m 68.6M - 5.7 3d0c579d3b62 c8ec68b2b94d 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (67s) 38s ago 13m 69.3M - 5.7 3d0c579d3b62 69ae06cbd222 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (49s) 11s ago 20m 8052k - 1.5.0 0da6a335fe13 e730b73491fd 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (42s) 38s ago 19m 3569k - 1.5.0 0da6a335fe13 ed32f4462b95 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 11s ago 18m 458M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 11s ago 18m 408M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:32:02.139 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 11s ago 17m 356M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:32:02.140 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 11s ago 17m 363M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:32:02.140 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (4m) 38s ago 16m 365M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:32:02.140 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (3m) 38s ago 16m 340M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:32:02.140 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (3m) 38s ago 16m 343M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:32:02.140 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (2m) 38s ago 15m 347M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:32:02.140 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (16s) 11s ago 19m 44.8M - 2.43.0 a07b618ecd1d f21a96b9c40e 2024-04-25T04:32:02.540 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:32:02.540 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:32:02.540 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:32:02.540 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:02.540 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 14 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:32:02.541 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:32:02.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:02 smithi078 ceph-mon[62942]: pgmap v370: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:02.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:02 smithi112 ceph-mon[50007]: pgmap v370: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:02.883 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "nfs", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "18/23 daemons upgraded", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:32:02.884 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:32:03.307 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:32:03.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:03 smithi078 ceph-mon[62942]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:03.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:03 smithi078 ceph-mon[62942]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:03.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:03 smithi078 ceph-mon[62942]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:03.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:03 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1673762808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:03.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:03 smithi112 ceph-mon[50007]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:03.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:03 smithi112 ceph-mon[50007]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:03.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:03 smithi112 ceph-mon[50007]: from='client.34616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:03.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:03 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1673762808' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:04.417 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:04 smithi078 ceph-mon[62942]: pgmap v371: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:04.417 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:04 smithi078 ceph-mon[62942]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:04.417 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:04 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/475377594' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:32:04.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:04 smithi112 ceph-mon[50007]: pgmap v371: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:04.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:04 smithi112 ceph-mon[50007]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:04.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:04 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/475377594' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:32:06.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:06 smithi078 ceph-mon[62942]: pgmap v372: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:06.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:06 smithi112 ceph-mon[50007]: pgmap v372: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:08.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:08 smithi078 ceph-mon[62942]: pgmap v373: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:08.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:08.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:08 smithi112 ceph-mon[50007]: pgmap v373: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:08.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:10.487 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:10 smithi078 ceph-mon[62942]: pgmap v374: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:10.487 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:10.488 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:10.488 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:10.488 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:10.488 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:10 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:10.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:10 smithi112 ceph-mon[50007]: pgmap v374: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:10.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:10.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:10.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:10.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:10.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:10 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:12.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:12 smithi078 ceph-mon[62942]: pgmap v375: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:12.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:12 smithi112 ceph-mon[50007]: pgmap v375: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:14.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:14 smithi078 ceph-mon[62942]: pgmap v376: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:14.649 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:14 smithi112 ceph-mon[50007]: pgmap v376: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:16.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:15 smithi078 ceph-mon[62942]: pgmap v377: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:16.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:15 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:16.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:15 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:16.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:15 smithi112 ceph-mon[50007]: pgmap v377: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:16.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:15 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:16.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:15 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:17.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:16 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:17.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:16 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:17.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:16 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:17.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:16 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:18.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:17 smithi078 ceph-mon[62942]: pgmap v378: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:18.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:17 smithi112 ceph-mon[50007]: pgmap v378: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:20.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: pgmap v379: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:20.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:19 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: pgmap v379: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:20.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:19 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:21.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:20 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:32:21.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:20 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-25T04:32:22.050 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:21 smithi112 ceph-mon[50007]: pgmap v380: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:22.050 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:21 smithi112 ceph-mon[50007]: Upgrade: Updating grafana.smithi078 2024-04-25T04:32:22.050 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:21 smithi112 ceph-mon[50007]: Deploying daemon grafana.smithi078 on smithi078 2024-04-25T04:32:22.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:21 smithi078 ceph-mon[62942]: pgmap v380: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:22.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:21 smithi078 ceph-mon[62942]: Upgrade: Updating grafana.smithi078 2024-04-25T04:32:22.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:21 smithi078 ceph-mon[62942]: Deploying daemon grafana.smithi078 on smithi078 2024-04-25T04:32:24.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:23 smithi078 ceph-mon[62942]: pgmap v381: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:24.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:24.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:23 smithi112 ceph-mon[50007]: pgmap v381: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:24.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:26.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:25 smithi078 ceph-mon[62942]: pgmap v382: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:26.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:25 smithi112 ceph-mon[50007]: pgmap v382: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:28.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:27 smithi078 ceph-mon[62942]: pgmap v383: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:28.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:27 smithi112 ceph-mon[50007]: pgmap v383: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:30.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:29 smithi078 ceph-mon[62942]: pgmap v384: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:30.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:29 smithi112 ceph-mon[50007]: pgmap v384: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:32.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:31 smithi078 ceph-mon[62942]: pgmap v385: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:32.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:31 smithi112 ceph-mon[50007]: pgmap v385: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:33.724 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:32:34.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:33 smithi078 ceph-mon[62942]: pgmap v386: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:34.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:34.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:34.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:33 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:34.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:33 smithi112 ceph-mon[50007]: pgmap v386: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:34.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:34.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:34.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:33 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (25s) 19s ago 21m 15.2M - 0.25.0 c8568f914cd2 0b68ed326705 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 19s ago 20m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (8m) 71s ago 20m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (2m) 19s ago 14m 36.5M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 479f42f132ed 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (119s) 71s ago 14m 12.4M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 3f760ad5208f 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (11m) 19s ago 22m 519M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (10m) 71s ago 20m 454M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:32:34.424 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 19s ago 22m 63.0M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (9m) 71s ago 20m 48.2M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (105s) 19s ago 14m 68.7M - 5.7 3d0c579d3b62 c8ec68b2b94d 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (99s) 71s ago 14m 69.3M - 5.7 3d0c579d3b62 69ae06cbd222 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (81s) 19s ago 20m 8006k - 1.5.0 0da6a335fe13 e730b73491fd 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (74s) 71s ago 19m 3569k - 1.5.0 0da6a335fe13 ed32f4462b95 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 19s ago 19m 458M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 19s ago 18m 409M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 19s ago 18m 356M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 19s ago 17m 364M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (4m) 71s ago 17m 365M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (4m) 71s ago 17m 340M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (3m) 71s ago 16m 343M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (2m) 71s ago 16m 347M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:32:34.425 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (48s) 19s ago 19m 49.0M - 2.43.0 a07b618ecd1d f21a96b9c40e 2024-04-25T04:32:34.831 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:32:34.831 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:32:34.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:32:34.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:34.831 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 14 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:32:34.832 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:32:35.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:34 smithi078 ceph-mon[62942]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:35.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:34 smithi078 ceph-mon[62942]: from='client.34636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:35.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:34 smithi112 ceph-mon[50007]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:35.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:34 smithi112 ceph-mon[50007]: from='client.34636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:35.197 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "nfs", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "18/23 daemons upgraded", 2024-04-25T04:32:35.198 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading grafana daemons", 2024-04-25T04:32:35.199 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:32:35.199 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:32:35.619 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:32:35.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:35 smithi078 ceph-mon[62942]: pgmap v387: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:35.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:35 smithi078 ceph-mon[62942]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:35.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:35 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1675615208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:35.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:35 smithi078 ceph-mon[62942]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:35.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:35 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3538286313' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:32:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:35 smithi112 ceph-mon[50007]: pgmap v387: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:35 smithi112 ceph-mon[50007]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:35 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1675615208' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:35 smithi112 ceph-mon[50007]: from='client.34648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:32:36.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:35 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3538286313' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:32:37.995 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:37 smithi078 ceph-mon[62942]: pgmap v388: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:38.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:37 smithi112 ceph-mon[50007]: pgmap v388: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:38.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:38.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:38.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:39.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:39.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:39.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:40.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:39 smithi078 ceph-mon[62942]: pgmap v389: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:40.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:40.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:40.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:39 smithi112 ceph-mon[50007]: pgmap v389: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:40.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:40.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:41.988 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:41 smithi078 ceph-mon[62942]: pgmap v390: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:42.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:41 smithi112 ceph-mon[50007]: pgmap v390: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:44.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: pgmap v391: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:44.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:44.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:44.149 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.150 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: Upgrade: Finalizing container_image settings 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-25T04:32:44.151 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-25T04:32:44.152 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: Upgrade: Complete! 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:44.153 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.154 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: pgmap v391: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: Upgrade: Finalizing container_image settings 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-25T04:32:44.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: Upgrade: Complete! 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:44.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:32:46.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:45 smithi112 ceph-mon[50007]: pgmap v392: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:46.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:45 smithi078 ceph-mon[62942]: pgmap v392: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:48.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:47 smithi112 ceph-mon[50007]: pgmap v393: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:48.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:47 smithi078 ceph-mon[62942]: pgmap v393: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:50.148 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:49 smithi112 ceph-mon[50007]: pgmap v394: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:50.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:49 smithi078 ceph-mon[62942]: pgmap v394: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:52.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:51 smithi078 ceph-mon[62942]: pgmap v395: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:52.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:51 smithi112 ceph-mon[50007]: pgmap v395: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:54.200 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:53 smithi112 ceph-mon[50007]: pgmap v396: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:54.201 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:53 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:54.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:53 smithi078 ceph-mon[62942]: pgmap v396: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:32:54.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:53 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:32:56.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:55 smithi078 ceph-mon[62942]: pgmap v397: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:56.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:55 smithi112 ceph-mon[50007]: pgmap v397: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:58.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:57 smithi078 ceph-mon[62942]: pgmap v398: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:32:58.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:57 smithi112 ceph-mon[50007]: pgmap v398: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:00.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:32:59 smithi078 ceph-mon[62942]: pgmap v399: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:00.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:32:59 smithi112 ceph-mon[50007]: pgmap v399: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:02.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:01 smithi078 ceph-mon[62942]: pgmap v400: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:02.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:01 smithi112 ceph-mon[50007]: pgmap v400: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:04.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:03 smithi078 ceph-mon[62942]: pgmap v401: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:04.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:03 smithi112 ceph-mon[50007]: pgmap v401: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:06.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:05 smithi078 ceph-mon[62942]: pgmap v402: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:06.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:05 smithi112 ceph-mon[50007]: pgmap v402: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:06.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph orch ps' 2024-04-25T04:33:07.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:06 smithi078 ceph-mon[62942]: from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:33:07.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:06 smithi112 ceph-mon[50007]: from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:33:08.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:07 smithi078 ceph-mon[62942]: pgmap v403: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:08.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:07 smithi112 ceph-mon[50007]: pgmap v403: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:08.630 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (59s) 30s ago 21m 15.3M - 0.25.0 c8568f914cd2 0b68ed326705 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 30s ago 21m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (9m) 105s ago 20m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (36s) 30s ago 21m 51.3M - 9.4.12 2bacad6d85d8 2a3b9f563fd7 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (3m) 30s ago 15m 36.7M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 479f42f132ed 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (2m) 105s ago 15m 12.4M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 3f760ad5208f 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (11m) 30s ago 22m 526M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (11m) 105s ago 20m 454M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 30s ago 22m 64.0M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (10m) 105s ago 20m 48.2M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (2m) 30s ago 15m 68.7M - 5.7 3d0c579d3b62 c8ec68b2b94d 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (2m) 105s ago 14m 69.3M - 5.7 3d0c579d3b62 69ae06cbd222 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (115s) 30s ago 21m 8330k - 1.5.0 0da6a335fe13 e730b73491fd 2024-04-25T04:33:08.631 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (109s) 105s ago 20m 3569k - 1.5.0 0da6a335fe13 ed32f4462b95 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 30s ago 19m 459M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 30s ago 19m 409M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 30s ago 18m 355M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 30s ago 18m 364M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (5m) 105s ago 18m 365M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (4m) 105s ago 17m 340M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (4m) 105s ago 17m 343M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (3m) 105s ago 16m 347M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:33:08.632 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (83s) 30s ago 20m 49.2M - 2.43.0 a07b618ecd1d f21a96b9c40e 2024-04-25T04:33:09.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:08 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:09.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph versions' 2024-04-25T04:33:09.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:08 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:10.280 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:09 smithi078 ceph-mon[62942]: pgmap v404: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:10.281 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:09 smithi078 ceph-mon[62942]: from='client.34660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:33:10.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:09 smithi112 ceph-mon[50007]: pgmap v404: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:10.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:09 smithi112 ceph-mon[50007]: from='client.34660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:33:11.314 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:33:11.314 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:33:11.314 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:33:11.314 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:33:11.314 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:33:11.314 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:33:11.314 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 14 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:33:11.315 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:33:11.955 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:11 smithi078 ceph-mon[62942]: pgmap v405: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:11.955 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:11 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2093267850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:33:11.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-25T04:33:12.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:11 smithi112 ceph-mon[50007]: pgmap v405: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:12.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:11 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2093267850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:33:13.726 INFO:teuthology.orchestra.run.smithi078.stdout:wait for servicemap items w/ changing names to refresh 2024-04-25T04:33:14.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:13 smithi078 ceph-mon[62942]: pgmap v406: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:14.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:13 smithi112 ceph-mon[50007]: pgmap v406: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:14.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'sleep 60' 2024-04-25T04:33:16.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:15 smithi078 ceph-mon[62942]: pgmap v407: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:16.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:15 smithi112 ceph-mon[50007]: pgmap v407: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:18.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:17 smithi078 ceph-mon[62942]: pgmap v408: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:18.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:17 smithi112 ceph-mon[50007]: pgmap v408: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:20.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:19 smithi078 ceph-mon[62942]: pgmap v409: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:20.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:19 smithi112 ceph-mon[50007]: pgmap v409: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:22.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:21 smithi078 ceph-mon[62942]: pgmap v410: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:22.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:21 smithi112 ceph-mon[50007]: pgmap v410: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:24.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:23 smithi078 ceph-mon[62942]: pgmap v411: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:24.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:23 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:24.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:23 smithi112 ceph-mon[50007]: pgmap v411: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:24.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:23 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:26.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:25 smithi078 ceph-mon[62942]: pgmap v412: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:26.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:25 smithi112 ceph-mon[50007]: pgmap v412: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:28.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:27 smithi078 ceph-mon[62942]: pgmap v413: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:28.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:27 smithi112 ceph-mon[50007]: pgmap v413: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:30.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:29 smithi078 ceph-mon[62942]: pgmap v414: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:30.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:29 smithi112 ceph-mon[50007]: pgmap v414: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:32.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:31 smithi078 ceph-mon[62942]: pgmap v415: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:32.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:31 smithi112 ceph-mon[50007]: pgmap v415: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:34.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:33 smithi078 ceph-mon[62942]: pgmap v416: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:34.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:33 smithi112 ceph-mon[50007]: pgmap v416: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:36.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:35 smithi078 ceph-mon[62942]: pgmap v417: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:36.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:35 smithi112 ceph-mon[50007]: pgmap v417: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:38.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:37 smithi078 ceph-mon[62942]: pgmap v418: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:38.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:37 smithi112 ceph-mon[50007]: pgmap v418: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:39.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:38 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:39.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:38 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:40.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:39 smithi078 ceph-mon[62942]: pgmap v419: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:40.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:39 smithi112 ceph-mon[50007]: pgmap v419: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:42.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:41 smithi078 ceph-mon[62942]: pgmap v420: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:42.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:41 smithi112 ceph-mon[50007]: pgmap v420: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:44.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:43 smithi078 ceph-mon[62942]: pgmap v421: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:44.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:33:44.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:33:44.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:33:44.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:43 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:33:44.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:43 smithi112 ceph-mon[50007]: pgmap v421: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:44.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:33:44.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:33:44.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:33:44.399 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:43 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:33:45.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:33:45.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:44 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:33:46.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:45 smithi078 ceph-mon[62942]: pgmap v422: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:46.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:45 smithi112 ceph-mon[50007]: pgmap v422: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:48.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:47 smithi078 ceph-mon[62942]: pgmap v423: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:48.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:47 smithi112 ceph-mon[50007]: pgmap v423: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:50.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:49 smithi078 ceph-mon[62942]: pgmap v424: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:50.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:49 smithi112 ceph-mon[50007]: pgmap v424: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:52.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:52 smithi078 ceph-mon[62942]: pgmap v425: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:52.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:52 smithi112 ceph-mon[50007]: pgmap v425: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:54.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:54 smithi078 ceph-mon[62942]: pgmap v426: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:54.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:54 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:54.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:54 smithi112 ceph-mon[50007]: pgmap v426: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:33:54.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:54 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:33:56.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:56 smithi078 ceph-mon[62942]: pgmap v427: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:56.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:56 smithi112 ceph-mon[50007]: pgmap v427: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:58.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:33:58 smithi078 ceph-mon[62942]: pgmap v428: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:33:58.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:33:58 smithi112 ceph-mon[50007]: pgmap v428: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:00.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:00 smithi078 ceph-mon[62942]: pgmap v429: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:00.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:00 smithi112 ceph-mon[50007]: pgmap v429: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:02.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:02 smithi078 ceph-mon[62942]: pgmap v430: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:02.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:02 smithi112 ceph-mon[50007]: pgmap v430: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:04.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:04 smithi078 ceph-mon[62942]: pgmap v431: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:04.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:04 smithi112 ceph-mon[50007]: pgmap v431: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:06.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:06 smithi078 ceph-mon[62942]: pgmap v432: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:06.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:06 smithi112 ceph-mon[50007]: pgmap v432: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:08.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:08 smithi078 ceph-mon[62942]: pgmap v433: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:08.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:08 smithi112 ceph-mon[50007]: pgmap v433: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:09.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:09 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:34:09.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:09 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:34:10.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:10 smithi078 ceph-mon[62942]: pgmap v434: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:10.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:10 smithi112 ceph-mon[50007]: pgmap v434: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:12.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:12 smithi078 ceph-mon[62942]: pgmap v435: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:12.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:12 smithi112 ceph-mon[50007]: pgmap v435: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:14.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:14 smithi078 ceph-mon[62942]: pgmap v436: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:14.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:14 smithi112 ceph-mon[50007]: pgmap v436: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:16.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:16 smithi078 ceph-mon[62942]: pgmap v437: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:16.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:16 smithi112 ceph-mon[50007]: pgmap v437: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:16.722 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph orch ps' 2024-04-25T04:34:18.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:18 smithi078 ceph-mon[62942]: pgmap v438: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:18.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:18 smithi112 ceph-mon[50007]: pgmap v438: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:18.686 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-25T04:34:18.686 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (2m) 100s ago 22m 15.3M - 0.25.0 c8568f914cd2 0b68ed326705 2024-04-25T04:34:18.686 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 100s ago 22m 6665k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5a514ea1c0ec 2024-04-25T04:34:18.686 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi112 smithi112 running (10m) 2m ago 22m 6660k - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d39d023df50a 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (106s) 100s ago 22m 51.3M - 9.4.12 2bacad6d85d8 2a3b9f563fd7 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.nlrtle smithi078 running (4m) 100s ago 16m 36.7M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 479f42f132ed 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi112.xminls smithi112 running (3m) 2m ago 16m 12.4M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 3f760ad5208f 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.eugwik smithi078 *:8443,9283,8765 running (13m) 100s ago 23m 526M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 84af49abb304 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi112.lvtlnw smithi112 *:8443,9283,8765 running (12m) 2m ago 21m 454M - 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 c89b75f0b48e 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 100s ago 24m 64.0M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 261e9ec88b00 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi112 smithi112 running (11m) 2m ago 21m 48.2M 2048M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 cf109ce58bbc 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.edyrsx smithi078 *:2049 running (3m) 100s ago 16m 68.7M - 5.7 3d0c579d3b62 c8ec68b2b94d 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi112.ouafxs smithi112 *:2049 running (3m) 2m ago 16m 69.3M - 5.7 3d0c579d3b62 69ae06cbd222 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 100s ago 22m 8330k - 1.5.0 0da6a335fe13 e730b73491fd 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi112 smithi112 *:9100 running (2m) 2m ago 21m 3569k - 1.5.0 0da6a335fe13 ed32f4462b95 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 100s ago 21m 459M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d2d996c1e21f 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 100s ago 20m 409M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 5d98f4fa8718 2024-04-25T04:34:18.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 100s ago 20m 355M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 ee973230cd51 2024-04-25T04:34:18.688 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 100s ago 19m 364M 1936M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 a9bbda282776 2024-04-25T04:34:18.688 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi112 running (6m) 2m ago 19m 365M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 77af6e06ee8b 2024-04-25T04:34:18.688 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi112 running (5m) 2m ago 18m 340M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 789440173ccc 2024-04-25T04:34:18.688 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi112 running (5m) 2m ago 18m 343M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 75c18bf56ed6 2024-04-25T04:34:18.688 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi112 running (4m) 2m ago 18m 347M 2704M 19.0.0-3293-gd6f5f4e5 3d0c579d3b62 d9186eec4bcc 2024-04-25T04:34:18.688 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (2m) 100s ago 21m 49.2M - 2.43.0 a07b618ecd1d f21a96b9c40e 2024-04-25T04:34:19.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph orch upgrade status' 2024-04-25T04:34:20.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:20 smithi078 ceph-mon[62942]: pgmap v439: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:20.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:20 smithi078 ceph-mon[62942]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:20.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:20 smithi112 ceph-mon[50007]: pgmap v439: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:20.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:20 smithi112 ceph-mon[50007]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:21.417 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:34:21.417 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-04-25T04:34:21.417 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-04-25T04:34:21.417 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-04-25T04:34:21.417 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-04-25T04:34:21.418 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-04-25T04:34:21.418 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-04-25T04:34:21.418 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-04-25T04:34:21.418 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:34:22.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph health detail' 2024-04-25T04:34:22.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:22 smithi078 ceph-mon[62942]: pgmap v440: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:22.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:22 smithi112 ceph-mon[50007]: pgmap v440: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:23.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:23 smithi078 ceph-mon[62942]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:23.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:23 smithi112 ceph-mon[50007]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:24.029 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-04-25T04:34:24.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:24 smithi078 ceph-mon[62942]: pgmap v441: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:24.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:24 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:34:24.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:24 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/3384039250' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:34:24.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:24 smithi112 ceph-mon[50007]: pgmap v441: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:24.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:24 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:34:24.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:24 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/3384039250' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-25T04:34:24.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph versions' 2024-04-25T04:34:26.225 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:26 smithi078 ceph-mon[62942]: pgmap v442: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:26.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:26 smithi112 ceph-mon[50007]: pgmap v442: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 8 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:34:26.715 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-04-25T04:34:26.716 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 2 2024-04-25T04:34:26.716 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:34:26.716 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-04-25T04:34:26.716 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)": 14 2024-04-25T04:34:26.716 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:34:26.716 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-04-25T04:34:27.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:27 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/145332617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:34:27.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-25T04:34:27.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:27 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/145332617' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:34:28.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:28 smithi078 ceph-mon[62942]: pgmap v443: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:28.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:28 smithi112 ceph-mon[50007]: pgmap v443: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:29.479 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-04-25T04:34:30.097 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -e sha1=d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-25T04:34:30.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:30 smithi078 ceph-mon[62942]: pgmap v444: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:30.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:30 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1519954539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:34:30.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:30 smithi112 ceph-mon[50007]: pgmap v444: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:30.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:30 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1519954539' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:34:32.242 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.0.0-3293-gd6f5f4e5 (d6f5f4e5ec08db21b2896cb71b4cb907b7fb4737) squid (dev)" 2024-04-25T04:34:32.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:32 smithi078 ceph-mon[62942]: pgmap v445: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:32.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:32 smithi112 ceph-mon[50007]: pgmap v445: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:32.848 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-04-25T04:34:32.848 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-04-25T04:34:32.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- ceph orch ls -f json 2024-04-25T04:34:33.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:33 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/1695735376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:34:33.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:33 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/1695735376' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-25T04:34:34.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:34 smithi078 ceph-mon[62942]: pgmap v446: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:34.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:34 smithi112 ceph-mon[50007]: pgmap v446: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-04-25T04:34:34.862 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-04-25T04:34:34.862 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-25T04:10:57.785438Z", "last_refresh": "2024-04-25T04:32:38.460764Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-25T04:10:52.278017Z", "last_refresh": "2024-04-25T04:31:23.144750Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-04-25T04:10:55.081385Z", "last_refresh": "2024-04-25T04:32:38.460939Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-25T04:17:37.851955Z", "last_refresh": "2024-04-25T04:31:23.145959Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-25T04:10:50.033060Z", "last_refresh": "2024-04-25T04:31:23.144938Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi112:172.21.15.112=smithi112"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-25T04:11:44.202951Z", "last_refresh": "2024-04-25T04:31:23.145085Z", "running": 2, "size": 2}}, {"events": ["2024-04-25T04:33:43.987407Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-04-25T04:18:00.200601Z", "last_refresh": "2024-04-25T04:31:23.146104Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-25T04:10:56.481409Z", "last_refresh": "2024-04-25T04:31:23.145225Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72", "last_refresh": "2024-04-25T04:31:23.145364Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-25T04:10:53.692363Z", "last_refresh": "2024-04-25T04:32:38.461091Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-25T04:34:35.540 INFO:tasks.cephadm:nfs.foo has 2/2 2024-04-25T04:34:35.541 DEBUG:teuthology.parallel:result is None 2024-04-25T04:34:35.541 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-25T04:34:35.550 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:34:35.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-04-25T04:34:35.577 INFO:teuthology.orchestra.run.smithi078.stderr:+ umount /mnt/foo 2024-04-25T04:34:35.589 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-25T04:34:35.598 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:34:35.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-04-25T04:34:36.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:36 smithi078 ceph-mon[62942]: pgmap v447: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:36.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:36 smithi078 ceph-mon[62942]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:34:36.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:36 smithi112 ceph-mon[50007]: pgmap v447: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:36.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:36 smithi112 ceph-mon[50007]: from='client.34692 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-25T04:34:37.621 INFO:teuthology.orchestra.run.smithi078.stdout: "foo" 2024-04-25T04:34:38.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:38 smithi078 ceph-mon[62942]: pgmap v448: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:38.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-04-25T04:34:38.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:38 smithi112 ceph-mon[50007]: pgmap v448: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:39.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:39 smithi078 ceph-mon[62942]: from='client.34696 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:39.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:39 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:34:39.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:39 smithi112 ceph-mon[50007]: from='client.34696 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:39.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:39 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-25T04:34:40.258 INFO:teuthology.orchestra.run.smithi078.stdout:[ 2024-04-25T04:34:40.258 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-04-25T04:34:40.258 INFO:teuthology.orchestra.run.smithi078.stdout: "access_type": "RW", 2024-04-25T04:34:40.258 INFO:teuthology.orchestra.run.smithi078.stdout: "clients": [], 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_id": "foo", 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "export_id": 1, 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "fsal": { 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "fs_name": "foofs", 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "CEPH", 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "user_id": "nfs.foo.1" 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "protocols": [ 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: 4 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "pseudo": "/fake", 2024-04-25T04:34:40.259 INFO:teuthology.orchestra.run.smithi078.stdout: "security_label": true, 2024-04-25T04:34:40.260 INFO:teuthology.orchestra.run.smithi078.stdout: "squash": "none", 2024-04-25T04:34:40.260 INFO:teuthology.orchestra.run.smithi078.stdout: "transports": [ 2024-04-25T04:34:40.260 INFO:teuthology.orchestra.run.smithi078.stdout: "TCP" 2024-04-25T04:34:40.260 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-04-25T04:34:40.260 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-04-25T04:34:40.260 INFO:teuthology.orchestra.run.smithi078.stdout:] 2024-04-25T04:34:40.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:40 smithi078 ceph-mon[62942]: pgmap v449: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:40.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:40 smithi112 ceph-mon[50007]: pgmap v449: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:40.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-04-25T04:34:41.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:41 smithi078 ceph-mon[62942]: from='client.34700 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:41.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:41 smithi112 ceph-mon[50007]: from='client.34700 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-25T04:34:42.126 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:42 smithi078 ceph-mon[62942]: pgmap v450: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:42.126 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:42 smithi078 ceph-mon[62942]: mgrmap e41: smithi078.eugwik(active, since 12m), standbys: smithi112.lvtlnw 2024-04-25T04:34:42.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:42 smithi112 ceph-mon[50007]: pgmap v450: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-25T04:34:42.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:42 smithi112 ceph-mon[50007]: mgrmap e41: smithi078.eugwik(active, since 12m), standbys: smithi112.lvtlnw 2024-04-25T04:34:42.738 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.1 2024-04-25T04:34:42.739 INFO:teuthology.orchestra.run.smithi078.stdout:foo grace 2024-04-25T04:34:42.739 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.0 2024-04-25T04:34:42.739 INFO:teuthology.orchestra.run.smithi078.stdout:foo conf-nfs.foo 2024-04-25T04:34:42.739 INFO:teuthology.orchestra.run.smithi078.stdout:foo export-1 2024-04-25T04:34:43.440 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-25T04:34:43.450 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:34:43.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 -- bash -c 'set -ex 2024-04-25T04:34:43.451 DEBUG:teuthology.orchestra.run.smithi078:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-04-25T04:34:43.451 DEBUG:teuthology.orchestra.run.smithi078:> ' 2024-04-25T04:34:44.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:44 smithi078 ceph-mon[62942]: pgmap v451: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-25T04:34:44.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:44 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:34:44.397 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:44 smithi112 ceph-mon[50007]: pgmap v451: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-25T04:34:44.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:44 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-25T04:34:45.260 INFO:teuthology.orchestra.run.smithi078.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-04-25T04:34:45.479 INFO:teuthology.orchestra.run.smithi078.stderr:+ [[ 6 -gt 2 ]] 2024-04-25T04:34:46.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-mon[62942]: pgmap v452: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-25T04:34:46.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:34:46.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:34:46.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:34:46.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-mon[62942]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:34:46.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-mon[62942]: from='client.? 172.21.15.78:0/2500343742' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-25T04:34:46.251 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-25T04:34:46.263 INFO:tasks.cephadm:Teardown begin 2024-04-25T04:34:46.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T04:34:46.292 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T04:34:46.324 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-25T04:34:46.324 DEBUG:teuthology.orchestra.run.smithi078:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-25T04:34:46.350 DEBUG:teuthology.orchestra.run.smithi112:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-25T04:34:46.381 INFO:tasks.cephadm:Stopping all daemons... 2024-04-25T04:34:46.381 INFO:tasks.cephadm.mon.smithi078:Stopping mon.smithi078... 2024-04-25T04:34:46.381 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078 2024-04-25T04:34:46.397 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:46 smithi112 ceph-mon[50007]: pgmap v452: 97 pgs: 97 active+clean; 225 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-25T04:34:46.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-25T04:34:46.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-25T04:34:46.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:34:46.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:46 smithi112 ceph-mon[50007]: from='mgr.34104 172.21.15.78:0/3502548651' entity='mgr.smithi078.eugwik' 2024-04-25T04:34:46.398 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:46 smithi112 ceph-mon[50007]: from='client.? 172.21.15.78:0/2500343742' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-25T04:34:46.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:34:47.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[62938]: 2024-04-25T04:34:46.705+0000 7f6f215ef700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:34:47.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078[62938]: 2024-04-25T04:34:46.705+0000 7f6f215ef700 -1 mon.smithi078@0(leader) e3 *** Got Signal Terminated *** 2024-04-25T04:34:47.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:46 smithi078 podman[109248]: 2024-04-25 04:34:46.778090717 +0000 UTC m=+0.203690436 container died 261e9ec88b00572e9f96cbd75ea322a28d6e8af4d669a88db2190ccf6e427276 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-25T04:34:47.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:47 smithi078 podman[109248]: 2024-04-25 04:34:47.189957211 +0000 UTC m=+0.615556937 container cleanup 261e9ec88b00572e9f96cbd75ea322a28d6e8af4d669a88db2190ccf6e427276 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:34:47.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Apr 25 04:34:47 smithi078 bash[109248]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi078 2024-04-25T04:34:47.566 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi078.service' 2024-04-25T04:34:47.614 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:34:47.615 INFO:tasks.cephadm.mon.smithi078:Stopped mon.smithi078 2024-04-25T04:34:47.615 INFO:tasks.cephadm.mon.smithi112:Stopping mon.smithi112... 2024-04-25T04:34:47.615 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi112 2024-04-25T04:34:47.898 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:47 smithi112 systemd[1]: Stopping Ceph mon.smithi112 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:34:48.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112[50003]: 2024-04-25T04:34:48.041+0000 7f776a0c0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi112 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:34:48.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112[50003]: 2024-04-25T04:34:48.041+0000 7f776a0c0700 -1 mon.smithi112@1(peon) e3 *** Got Signal Terminated *** 2024-04-25T04:34:48.354 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:48 smithi112 podman[76411]: 2024-04-25 04:34:48.072959734 +0000 UTC m=+0.261507068 container died cf109ce58bbce47b69b86cf9f1123581e047845b354ccae8611cd9f3e9baa214 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:34:48.647 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:48 smithi112 podman[76411]: 2024-04-25 04:34:48.509248797 +0000 UTC m=+0.697796119 container cleanup cf109ce58bbce47b69b86cf9f1123581e047845b354ccae8611cd9f3e9baa214 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux ) 2024-04-25T04:34:48.648 INFO:journalctl@ceph.mon.smithi112.smithi112.stdout:Apr 25 04:34:48 smithi112 bash[76411]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-mon-smithi112 2024-04-25T04:34:48.941 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@mon.smithi112.service' 2024-04-25T04:34:48.993 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:34:48.994 INFO:tasks.cephadm.mon.smithi112:Stopped mon.smithi112 2024-04-25T04:34:48.994 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-25T04:34:48.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.0 2024-04-25T04:34:49.288 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:49 smithi078 systemd[1]: Stopping Ceph osd.0 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:34:49.602 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:49 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[69128]: 2024-04-25T04:34:49.284+0000 7f967ef82700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:34:49.603 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:49 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[69128]: 2024-04-25T04:34:49.284+0000 7f967ef82700 -1 osd.0 161 *** Got signal Terminated *** 2024-04-25T04:34:49.603 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:49 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0[69128]: 2024-04-25T04:34:49.284+0000 7f967ef82700 -1 osd.0 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:34:55.102 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:54 smithi078 podman[109380]: 2024-04-25 04:34:54.794853541 +0000 UTC m=+5.611782054 container died d2d996c1e21f9419b11c53dcfa10ee59809704f2ed5332742682cba23332d77d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-25T04:34:55.557 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:55 smithi078 podman[109380]: 2024-04-25 04:34:55.312001651 +0000 UTC m=+6.128930197 container cleanup d2d996c1e21f9419b11c53dcfa10ee59809704f2ed5332742682cba23332d77d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-25T04:34:55.557 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:55 smithi078 bash[109380]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0 2024-04-25T04:34:55.852 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:55 smithi078 podman[109429]: 2024-04-25 04:34:55.55516119 +0000 UTC m=+0.752209653 container remove d2d996c1e21f9419b11c53dcfa10ee59809704f2ed5332742682cba23332d77d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-25T04:34:56.139 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:55 smithi078 podman[109522]: 2024-04-25 04:34:55.789492261 +0000 UTC m=+0.018365535 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:34:56.140 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:55 smithi078 podman[109522]: 2024-04-25 04:34:55.889585391 +0000 UTC m=+0.118458683 container create 6696d7940dd79c75c96bc9949c9275fafaab67ca86e89b8fee8a2e3d4afc0a92 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-25T04:34:56.140 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:56 smithi078 podman[109522]: 2024-04-25 04:34:56.133968414 +0000 UTC m=+0.362841667 container init 6696d7940dd79c75c96bc9949c9275fafaab67ca86e89b8fee8a2e3d4afc0a92 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:34:56.437 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:56 smithi078 podman[109522]: 2024-04-25 04:34:56.137338516 +0000 UTC m=+0.366211766 container start 6696d7940dd79c75c96bc9949c9275fafaab67ca86e89b8fee8a2e3d4afc0a92 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-25T04:34:56.437 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:56 smithi078 podman[109522]: 2024-04-25 04:34:56.242530123 +0000 UTC m=+0.471403400 container attach 6696d7940dd79c75c96bc9949c9275fafaab67ca86e89b8fee8a2e3d4afc0a92 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-25T04:34:56.749 INFO:journalctl@ceph.osd.0.smithi078.stdout:Apr 25 04:34:56 smithi078 podman[109522]: 2024-04-25 04:34:56.435147349 +0000 UTC m=+0.664020592 container died 6696d7940dd79c75c96bc9949c9275fafaab67ca86e89b8fee8a2e3d4afc0a92 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-25T04:34:57.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.0.service' 2024-04-25T04:34:57.116 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:34:57.117 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-25T04:34:57.117 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-25T04:34:57.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.1 2024-04-25T04:34:57.352 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:34:57 smithi078 systemd[1]: Stopping Ceph osd.1 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:34:57.852 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:34:57 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:34:57.457+0000 7f53ecb8c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:34:57.852 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:34:57 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:34:57.457+0000 7f53ecb8c700 -1 osd.1 161 *** Got signal Terminated *** 2024-04-25T04:34:57.852 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:34:57 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1[71672]: 2024-04-25T04:34:57.457+0000 7f53ecb8c700 -1 osd.1 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:35:03.102 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:02 smithi078 podman[109629]: 2024-04-25 04:35:02.84370657 +0000 UTC m=+5.505447592 container died 5d98f4fa8718fa5c046f802f5292890b4f990bbf3fcaf0dc62c8bf800446e8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:35:03.503 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:03 smithi078 podman[109629]: 2024-04-25 04:35:03.27500073 +0000 UTC m=+5.936741759 container cleanup 5d98f4fa8718fa5c046f802f5292890b4f990bbf3fcaf0dc62c8bf800446e8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , ceph=True) 2024-04-25T04:35:03.504 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:03 smithi078 bash[109629]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1 2024-04-25T04:35:03.852 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:03 smithi078 podman[109648]: 2024-04-25 04:35:03.501560486 +0000 UTC m=+0.649987049 container remove 5d98f4fa8718fa5c046f802f5292890b4f990bbf3fcaf0dc62c8bf800446e8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True) 2024-04-25T04:35:04.266 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:03 smithi078 podman[109741]: 2024-04-25 04:35:03.936183617 +0000 UTC m=+0.110444163 container create 7f04555ed4978177c713b099f21fd787b788ce35c0b68314d16430ffd7f4bcef (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-25T04:35:04.266 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:03 smithi078 podman[109741]: 2024-04-25 04:35:03.843930103 +0000 UTC m=+0.018190620 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:35:04.266 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:04 smithi078 podman[109741]: 2024-04-25 04:35:04.180384631 +0000 UTC m=+0.354645134 container init 7f04555ed4978177c713b099f21fd787b788ce35c0b68314d16430ffd7f4bcef (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-25T04:35:04.266 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:04 smithi078 podman[109741]: 2024-04-25 04:35:04.183556133 +0000 UTC m=+0.357816645 container start 7f04555ed4978177c713b099f21fd787b788ce35c0b68314d16430ffd7f4bcef (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-25T04:35:04.266 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:04 smithi078 podman[109741]: 2024-04-25 04:35:04.263549643 +0000 UTC m=+0.437810146 container attach 7f04555ed4978177c713b099f21fd787b788ce35c0b68314d16430ffd7f4bcef (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-25T04:35:04.602 INFO:journalctl@ceph.osd.1.smithi078.stdout:Apr 25 04:35:04 smithi078 podman[109741]: 2024-04-25 04:35:04.481122408 +0000 UTC m=+0.655382903 container died 7f04555ed4978177c713b099f21fd787b788ce35c0b68314d16430ffd7f4bcef (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD) 2024-04-25T04:35:05.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.1.service' 2024-04-25T04:35:05.157 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:35:05.157 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-25T04:35:05.157 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-25T04:35:05.157 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.2 2024-04-25T04:35:05.352 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:05 smithi078 systemd[1]: Stopping Ceph osd.2 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:35:05.852 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:05 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[74448]: 2024-04-25T04:35:05.519+0000 7fe052d16700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:35:05.852 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:05 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[74448]: 2024-04-25T04:35:05.519+0000 7fe052d16700 -1 osd.2 161 *** Got signal Terminated *** 2024-04-25T04:35:05.853 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:05 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2[74448]: 2024-04-25T04:35:05.519+0000 7fe052d16700 -1 osd.2 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:35:11.411 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:11 smithi078 podman[109846]: 2024-04-25 04:35:11.109977807 +0000 UTC m=+5.724461179 container died ee973230cd51542cb98c14eb3538fd2b4c899b3d8ac0c32cd49d11ec37c1fa21 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-25T04:35:11.760 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:11 smithi078 podman[109846]: 2024-04-25 04:35:11.514724965 +0000 UTC m=+6.129208337 container cleanup ee973230cd51542cb98c14eb3538fd2b4c899b3d8ac0c32cd49d11ec37c1fa21 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-25T04:35:11.760 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:11 smithi078 bash[109846]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2 2024-04-25T04:35:12.055 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:11 smithi078 podman[109865]: 2024-04-25 04:35:11.757666226 +0000 UTC m=+0.644510750 container remove ee973230cd51542cb98c14eb3538fd2b4c899b3d8ac0c32cd49d11ec37c1fa21 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-25T04:35:12.314 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:12 smithi078 podman[109961]: 2024-04-25 04:35:11.956424485 +0000 UTC m=+0.018426966 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:35:12.314 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:12 smithi078 podman[109961]: 2024-04-25 04:35:12.058485314 +0000 UTC m=+0.120487823 container create 6dd6d75ca136dc6341f179d2e7eebbad3a802c3d575a8b96dbea3b9de692dcce (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-04-25T04:35:12.314 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:12 smithi078 podman[109961]: 2024-04-25 04:35:12.312138664 +0000 UTC m=+0.374141131 container init 6dd6d75ca136dc6341f179d2e7eebbad3a802c3d575a8b96dbea3b9de692dcce (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:12.602 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:12 smithi078 podman[109961]: 2024-04-25 04:35:12.315503633 +0000 UTC m=+0.377506101 container start 6dd6d75ca136dc6341f179d2e7eebbad3a802c3d575a8b96dbea3b9de692dcce (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:12.602 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:12 smithi078 podman[109961]: 2024-04-25 04:35:12.411683886 +0000 UTC m=+0.473686353 container attach 6dd6d75ca136dc6341f179d2e7eebbad3a802c3d575a8b96dbea3b9de692dcce (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:35:12.892 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:12 smithi078 podman[109961]: 2024-04-25 04:35:12.604330403 +0000 UTC m=+0.666332867 container died 6dd6d75ca136dc6341f179d2e7eebbad3a802c3d575a8b96dbea3b9de692dcce (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-25T04:35:13.144 INFO:journalctl@ceph.osd.2.smithi078.stdout:Apr 25 04:35:13 smithi078 podman[109987]: 2024-04-25 04:35:13.140082394 +0000 UTC m=+0.530420945 container remove 6dd6d75ca136dc6341f179d2e7eebbad3a802c3d575a8b96dbea3b9de692dcce (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-25T04:35:13.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.2.service' 2024-04-25T04:35:13.197 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:35:13.198 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-25T04:35:13.198 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-25T04:35:13.198 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.3 2024-04-25T04:35:13.520 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:13 smithi078 systemd[1]: Stopping Ceph osd.3 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:35:13.852 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:13 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[76998]: 2024-04-25T04:35:13.517+0000 7f209296a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:35:13.852 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:13 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[76998]: 2024-04-25T04:35:13.517+0000 7f209296a700 -1 osd.3 161 *** Got signal Terminated *** 2024-04-25T04:35:13.852 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:13 smithi078 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3[76998]: 2024-04-25T04:35:13.517+0000 7f209296a700 -1 osd.3 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:35:16.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:16 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:16.049+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:17.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:17 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:17.001+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:17.545 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:17 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:17.160+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:18.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:18 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:18.125+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:18.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:18 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:17.999+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:19.397 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:18 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:18.955+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:19.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:19 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:19.113+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:19.430 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:19 smithi078 podman[110100]: 2024-04-25 04:35:19.121986878 +0000 UTC m=+5.701902561 container died a9bbda282776b2362a262b861649957f1db4d448043dde132c185d94c4177343 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-04-25T04:35:19.852 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:19 smithi078 podman[110100]: 2024-04-25 04:35:19.593254721 +0000 UTC m=+6.173170403 container cleanup a9bbda282776b2362a262b861649957f1db4d448043dde132c185d94c4177343 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-25T04:35:19.852 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:19 smithi078 bash[110100]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3 2024-04-25T04:35:20.216 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:19 smithi078 podman[110120]: 2024-04-25 04:35:19.877883752 +0000 UTC m=+0.746817544 container remove a9bbda282776b2362a262b861649957f1db4d448043dde132c185d94c4177343 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-25T04:35:20.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:20 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:20.066+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:20.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:19 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:19.950+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:20.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:19 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:19.919+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:20.558 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:20 smithi078 podman[110209]: 2024-04-25 04:35:20.117269047 +0000 UTC m=+0.018413473 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:35:20.558 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:20 smithi078 podman[110209]: 2024-04-25 04:35:20.220378887 +0000 UTC m=+0.121523330 container create b0e1bf11c613f2c40faa2882d468ad0ac5f481292e373d22b3c5c80954ad14d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:20.559 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:20 smithi078 podman[110209]: 2024-04-25 04:35:20.456435622 +0000 UTC m=+0.357580023 container init b0e1bf11c613f2c40faa2882d468ad0ac5f481292e373d22b3c5c80954ad14d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:20.559 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:20 smithi078 podman[110209]: 2024-04-25 04:35:20.459563518 +0000 UTC m=+0.360707923 container start b0e1bf11c613f2c40faa2882d468ad0ac5f481292e373d22b3c5c80954ad14d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True) 2024-04-25T04:35:20.559 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:20 smithi078 podman[110209]: 2024-04-25 04:35:20.556322587 +0000 UTC m=+0.457467033 container attach b0e1bf11c613f2c40faa2882d468ad0ac5f481292e373d22b3c5c80954ad14d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-25T04:35:20.852 INFO:journalctl@ceph.osd.3.smithi078.stdout:Apr 25 04:35:20 smithi078 podman[110209]: 2024-04-25 04:35:20.754137983 +0000 UTC m=+0.655282393 container died b0e1bf11c613f2c40faa2882d468ad0ac5f481292e373d22b3c5c80954ad14d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-25T04:35:21.147 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:20 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:20.982+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:21.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:20 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:20.842+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:48.800312+0000 front 2024-04-25T04:34:48.800383+0000 (oldest deadline 2024-04-25T04:35:20.000720+0000) 2024-04-25T04:35:21.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:20 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:20.951+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:21.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:21.033+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:21.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.3.service' 2024-04-25T04:35:21.484 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:35:21.484 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-25T04:35:21.484 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-25T04:35:21.485 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.4 2024-04-25T04:35:21.780 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:21 smithi112 systemd[1]: Stopping Ceph osd.4 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:35:22.147 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:21.991+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:22.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:21.937+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:22.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:21.970+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:22.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:21.777+0000 7f184f0c4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:35:22.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:21.777+0000 7f184f0c4700 -1 osd.4 161 *** Got signal Terminated *** 2024-04-25T04:35:22.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:21.777+0000 7f184f0c4700 -1 osd.4 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:35:22.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:21 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:21.831+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:48.800312+0000 front 2024-04-25T04:34:48.800383+0000 (oldest deadline 2024-04-25T04:35:20.000720+0000) 2024-04-25T04:35:23.147 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:22.979+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:23.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:22.904+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:23.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:22.793+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:48.800312+0000 front 2024-04-25T04:34:48.800383+0000 (oldest deadline 2024-04-25T04:35:20.000720+0000) 2024-04-25T04:35:23.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:22 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:22.937+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:24.025 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:23 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:23.771+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:48.800312+0000 front 2024-04-25T04:34:48.800383+0000 (oldest deadline 2024-04-25T04:35:20.000720+0000) 2024-04-25T04:35:24.025 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:23 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:23.903+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:24.025 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:23 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:23.923+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:24.397 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:24 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:24.022+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:25.147 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:24 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:24.992+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:25.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:24 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:24.954+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:25.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:24 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:24.746+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:48.800312+0000 front 2024-04-25T04:34:48.800383+0000 (oldest deadline 2024-04-25T04:35:20.000720+0000) 2024-04-25T04:35:25.149 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:24 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:24.904+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:25.998 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:25 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:25.879+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:25.998 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:25 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:25.976+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:25.998 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:25 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:25.728+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:48.800312+0000 front 2024-04-25T04:34:48.800383+0000 (oldest deadline 2024-04-25T04:35:20.000720+0000) 2024-04-25T04:35:25.998 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:25 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:25.728+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.001335+0000 front 2024-04-25T04:35:00.001412+0000 (oldest deadline 2024-04-25T04:35:25.301196+0000) 2024-04-25T04:35:26.397 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:25 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:25.995+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:26.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:25 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:25.995+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:27.050 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:26.908+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:27.050 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:26.908+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:27.051 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:26.766+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:48.800312+0000 front 2024-04-25T04:34:48.800383+0000 (oldest deadline 2024-04-25T04:35:20.000720+0000) 2024-04-25T04:35:27.051 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4[54949]: 2024-04-25T04:35:26.766+0000 7f184b6b8700 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.001335+0000 front 2024-04-25T04:35:00.001412+0000 (oldest deadline 2024-04-25T04:35:25.301196+0000) 2024-04-25T04:35:27.051 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:26.989+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:27.051 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:26.989+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:27.051 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:26.948+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:27.051 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:26 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:26.948+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:27.647 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:27 smithi112 podman[76609]: 2024-04-25 04:35:27.361538981 +0000 UTC m=+5.684081736 container died 77af6e06ee8bc006cb4eec855b4850cda78e08eb7766900b1426e473c97c6b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-25T04:35:27.919 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:27 smithi112 podman[76609]: 2024-04-25 04:35:27.828855866 +0000 UTC m=+6.151398670 container cleanup 77af6e06ee8bc006cb4eec855b4850cda78e08eb7766900b1426e473c97c6b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-25T04:35:27.919 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:27 smithi112 bash[76609]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4 2024-04-25T04:35:28.398 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:28 smithi112 podman[76627]: 2024-04-25 04:35:28.113823344 +0000 UTC m=+0.752444561 container remove 77af6e06ee8bc006cb4eec855b4850cda78e08eb7766900b1426e473c97c6b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:28.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:27 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:27.916+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:28.399 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:27 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:27.916+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:28.399 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:27 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:27.944+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:28.399 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:27 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:27.944+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:28.399 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:27 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:27.928+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:28.399 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:27 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:27.928+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:28.702 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:28 smithi112 podman[76723]: 2024-04-25 04:35:28.39791503 +0000 UTC m=+0.105692072 container create 6008e01dcf17ac4a2835c632bb0356b869e6c899c38197b0306bc754c6b34ac6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-25T04:35:28.703 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:28 smithi112 podman[76723]: 2024-04-25 04:35:28.310602566 +0000 UTC m=+0.018379632 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:35:28.703 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:28 smithi112 podman[76723]: 2024-04-25 04:35:28.625493614 +0000 UTC m=+0.333270666 container init 6008e01dcf17ac4a2835c632bb0356b869e6c899c38197b0306bc754c6b34ac6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-25T04:35:28.703 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:28 smithi112 podman[76723]: 2024-04-25 04:35:28.628507562 +0000 UTC m=+0.336284613 container start 6008e01dcf17ac4a2835c632bb0356b869e6c899c38197b0306bc754c6b34ac6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-25T04:35:28.703 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:28 smithi112 podman[76723]: 2024-04-25 04:35:28.700447418 +0000 UTC m=+0.408224495 container attach 6008e01dcf17ac4a2835c632bb0356b869e6c899c38197b0306bc754c6b34ac6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424) 2024-04-25T04:35:29.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:28 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:28.906+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:29.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:28 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:28.906+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:29.148 INFO:journalctl@ceph.osd.4.smithi112.stdout:Apr 25 04:35:28 smithi112 podman[76723]: 2024-04-25 04:35:28.924976709 +0000 UTC m=+0.632753763 container died 6008e01dcf17ac4a2835c632bb0356b869e6c899c38197b0306bc754c6b34ac6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-25T04:35:29.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:28 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:28.938+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:29.149 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:28 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:28.938+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:29.149 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:28 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:28.945+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:29.149 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:28 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:28.945+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:29.559 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.4.service' 2024-04-25T04:35:29.599 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:35:29.600 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-25T04:35:29.600 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-25T04:35:29.600 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.5 2024-04-25T04:35:29.807 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:29 smithi112 systemd[1]: Stopping Ceph osd.5 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:35:30.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:29.863+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:30.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:29.863+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:30.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:29.946+0000 7f49fbe8b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:35:30.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:29.946+0000 7f49fbe8b700 -1 osd.5 161 *** Got signal Terminated *** 2024-04-25T04:35:30.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:29.946+0000 7f49fbe8b700 -1 osd.5 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:35:30.149 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:29.896+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:30.149 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:29.896+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:30.149 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:29.984+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:30.149 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:29 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:29.984+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:31.147 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:30 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:30.933+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:31.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:30 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:30.933+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:31.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:30 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:30.868+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:31.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:30 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:30.868+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:31.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:30 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:30.948+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:31.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:30 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:30.948+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:32.147 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:31.909+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:32.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:31.909+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:32.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:31.909+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:32.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:31.878+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:32.148 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:31.878+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:32.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:31.949+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:32.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:31.949+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:32.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:31 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:31.949+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:33.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:32.907+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:33.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:32.907+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:33.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:32.945+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:33.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:32.945+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:33.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:32.945+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:33.399 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:32.950+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:33.399 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:32.950+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:33.399 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:32 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:32.950+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:34.397 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:33.978+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:34.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:33.978+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:34.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:33.978+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:34.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:33.938+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:34.398 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:33.938+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:34.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:33.953+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:34.399 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:33.953+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:34.399 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:33 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:33.953+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:35.349 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:34.904+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:35.349 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:34.904+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:35.349 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:34.904+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:35.349 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:34.915+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:51.416251+0000 front 2024-04-25T04:34:51.416179+0000 (oldest deadline 2024-04-25T04:35:15.416675+0000) 2024-04-25T04:35:35.349 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:34.915+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:01.317802+0000 front 2024-04-25T04:35:01.317813+0000 (oldest deadline 2024-04-25T04:35:26.017603+0000) 2024-04-25T04:35:35.349 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5[57222]: 2024-04-25T04:35:34.915+0000 7f49f847f700 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:10.118557+0000 front 2024-04-25T04:35:10.118538+0000 (oldest deadline 2024-04-25T04:35:34.818486+0000) 2024-04-25T04:35:35.350 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:34.998+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:35.350 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:34.998+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:35.350 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:34 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:34.998+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:35.633 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:35 smithi112 podman[76828]: 2024-04-25 04:35:35.346763918 +0000 UTC m=+5.513792849 container died 789440173ccc330489d30192918e85d63579bfdb2b9e9e0aefd3652fd0167c7d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=) 2024-04-25T04:35:35.897 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:35 smithi112 podman[76828]: 2024-04-25 04:35:35.85528444 +0000 UTC m=+6.022313425 container cleanup 789440173ccc330489d30192918e85d63579bfdb2b9e9e0aefd3652fd0167c7d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-25T04:35:35.898 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:35 smithi112 bash[76828]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5 2024-04-25T04:35:36.385 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:36 smithi112 podman[76888]: 2024-04-25 04:35:36.073588401 +0000 UTC m=+0.727583444 container remove 789440173ccc330489d30192918e85d63579bfdb2b9e9e0aefd3652fd0167c7d (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-25T04:35:36.386 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:35 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:35.919+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:36.386 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:35 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:35.919+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:36.386 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:35 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:35.919+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:36.386 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:36 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:36.016+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:36.386 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:36 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:36.016+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:36.386 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:36 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:36.016+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:36.648 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:36 smithi112 podman[76981]: 2024-04-25 04:35:36.28609781 +0000 UTC m=+0.018373522 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:35:36.648 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:36 smithi112 podman[76981]: 2024-04-25 04:35:36.405059024 +0000 UTC m=+0.137334722 container create a833e6d344b293d54c006236a6e8f72c30deea9e0a23c611d91c2e5c1f08d53f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-25T04:35:36.648 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:36 smithi112 podman[76981]: 2024-04-25 04:35:36.602722562 +0000 UTC m=+0.334998261 container init a833e6d344b293d54c006236a6e8f72c30deea9e0a23c611d91c2e5c1f08d53f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True) 2024-04-25T04:35:36.648 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:36 smithi112 podman[76981]: 2024-04-25 04:35:36.606138739 +0000 UTC m=+0.338414447 container start a833e6d344b293d54c006236a6e8f72c30deea9e0a23c611d91c2e5c1f08d53f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-25T04:35:36.958 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:36 smithi112 podman[76981]: 2024-04-25 04:35:36.676926985 +0000 UTC m=+0.409202703 container attach a833e6d344b293d54c006236a6e8f72c30deea9e0a23c611d91c2e5c1f08d53f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-25T04:35:36.958 INFO:journalctl@ceph.osd.5.smithi112.stdout:Apr 25 04:35:36 smithi112 podman[76981]: 2024-04-25 04:35:36.901134939 +0000 UTC m=+0.633410638 container died a833e6d344b293d54c006236a6e8f72c30deea9e0a23c611d91c2e5c1f08d53f (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e) 2024-04-25T04:35:37.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:36 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:36.953+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:37.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:36 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:36.953+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:37.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:36 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:36.953+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:37.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:37.055+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:37.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:37.055+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:37.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:37.055+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:37.482 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.5.service' 2024-04-25T04:35:37.522 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:35:37.522 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-25T04:35:37.522 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-25T04:35:37.522 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.6 2024-04-25T04:35:37.845 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:37 smithi112 systemd[1]: Stopping Ceph osd.6 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:35:38.147 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:38.000+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:38.000+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:38.000+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:37.841+0000 7f9ed5361700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:37.841+0000 7f9ed5361700 -1 osd.6 161 *** Got signal Terminated *** 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:37 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:37.841+0000 7f9ed5361700 -1 osd.6 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:38.051+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:38.051+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:38.148 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:38.051+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:39.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:39.025+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:39.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:39.025+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:39.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:39.025+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:39.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:38.973+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:39.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:38.973+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:39.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:38 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:38.973+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:40.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:40 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:40.068+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:40.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:40 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:40.068+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:40.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:40 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:40.068+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:40.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:39.986+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:40.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:39.986+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:40.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:39 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:39.986+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:41.081 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:41.035+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:41.081 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:41.035+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:41.082 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:41.035+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:41.397 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:41.078+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:41.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:41.078+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:41.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:41.078+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:42.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:42 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:42.070+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:52.045593+0000 front 2024-04-25T04:34:52.045722+0000 (oldest deadline 2024-04-25T04:35:16.745547+0000) 2024-04-25T04:35:42.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:42 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:42.070+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.147247+0000 front 2024-04-25T04:35:00.147229+0000 (oldest deadline 2024-04-25T04:35:26.047109+0000) 2024-04-25T04:35:42.398 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:42 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6[59485]: 2024-04-25T04:35:42.070+0000 7f9ecd94d700 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:07.748006+0000 front 2024-04-25T04:35:07.748064+0000 (oldest deadline 2024-04-25T04:35:31.247759+0000) 2024-04-25T04:35:42.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:41.993+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:42.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:41.993+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:42.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:41 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:41.993+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:43.381 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:43 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:43.020+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:43.381 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:43 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:43.020+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:43.381 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:43 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:43.020+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:43.637 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:43 smithi112 podman[77089]: 2024-04-25 04:35:43.379236478 +0000 UTC m=+5.638990686 container died 75c18bf56ed69bb377517cf7feaef6809b778c230a0e7e88329f62760289942c (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-25T04:35:43.898 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:43 smithi112 podman[77089]: 2024-04-25 04:35:43.817199503 +0000 UTC m=+6.076953715 container cleanup 75c18bf56ed69bb377517cf7feaef6809b778c230a0e7e88329f62760289942c (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True) 2024-04-25T04:35:43.898 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:43 smithi112 bash[77089]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6 2024-04-25T04:35:44.389 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:43 smithi112 podman[77108]: 2024-04-25 04:35:43.985232463 +0000 UTC m=+0.609492071 container remove 75c18bf56ed69bb377517cf7feaef6809b778c230a0e7e88329f62760289942c (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-25T04:35:44.389 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:43 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:43.998+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:44.389 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:43 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:43.998+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:44.390 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:43 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:43.998+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:44.390 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:43 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:43.998+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-04-25T04:35:17.345770+0000 front 2024-04-25T04:35:17.345779+0000 (oldest deadline 2024-04-25T04:35:43.146130+0000) 2024-04-25T04:35:44.641 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:44 smithi112 podman[77199]: 2024-04-25 04:35:44.290526182 +0000 UTC m=+0.018173130 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:35:44.641 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:44 smithi112 podman[77199]: 2024-04-25 04:35:44.402672042 +0000 UTC m=+0.130318980 container create 81675e3e6827ab08610d812cb70eb4dca570d51b43f5de6c5e454c98262dcce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-25T04:35:44.641 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:44 smithi112 podman[77199]: 2024-04-25 04:35:44.638806646 +0000 UTC m=+0.366453581 container init 81675e3e6827ab08610d812cb70eb4dca570d51b43f5de6c5e454c98262dcce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-25T04:35:44.898 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:44 smithi112 podman[77199]: 2024-04-25 04:35:44.641961989 +0000 UTC m=+0.369608925 container start 81675e3e6827ab08610d812cb70eb4dca570d51b43f5de6c5e454c98262dcce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-25T04:35:44.898 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:44 smithi112 podman[77199]: 2024-04-25 04:35:44.722179832 +0000 UTC m=+0.449826768 container attach 81675e3e6827ab08610d812cb70eb4dca570d51b43f5de6c5e454c98262dcce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-25T04:35:45.210 INFO:journalctl@ceph.osd.6.smithi112.stdout:Apr 25 04:35:44 smithi112 podman[77199]: 2024-04-25 04:35:44.928379523 +0000 UTC m=+0.656026459 container died 81675e3e6827ab08610d812cb70eb4dca570d51b43f5de6c5e454c98262dcce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=) 2024-04-25T04:35:45.211 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:44.984+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:45.211 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:44.984+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:45.211 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:44.984+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:45.211 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:44 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:44.984+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-04-25T04:35:17.345770+0000 front 2024-04-25T04:35:17.345779+0000 (oldest deadline 2024-04-25T04:35:43.146130+0000) 2024-04-25T04:35:45.480 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.6.service' 2024-04-25T04:35:45.516 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:35:45.516 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-25T04:35:45.516 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-25T04:35:45.516 DEBUG:teuthology.orchestra.run.smithi112:> sudo systemctl stop ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.7 2024-04-25T04:35:45.831 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 systemd[1]: Stopping Ceph osd.7 for 9a8d9f72-02b9-11ef-bc93-c7b262605968... 2024-04-25T04:35:46.147 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:45.829+0000 7fdab5c5d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-25T04:35:46.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:45.829+0000 7fdab5c5d700 -1 osd.7 161 *** Got signal Terminated *** 2024-04-25T04:35:46.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:45.829+0000 7fdab5c5d700 -1 osd.7 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-25T04:35:46.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:45.965+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:46.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:45.965+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:46.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:45.965+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:46.148 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:45 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:45.965+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-04-25T04:35:17.345770+0000 front 2024-04-25T04:35:17.345779+0000 (oldest deadline 2024-04-25T04:35:43.146130+0000) 2024-04-25T04:35:47.395 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:46 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:46.952+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:47.446 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:46 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:46.952+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:47.446 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:46 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:46.952+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:47.446 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:46 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:46.952+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-04-25T04:35:17.345770+0000 front 2024-04-25T04:35:17.345779+0000 (oldest deadline 2024-04-25T04:35:43.146130+0000) 2024-04-25T04:35:48.295 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:47 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:47.980+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:48.295 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:47 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:47.980+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:48.296 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:47 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:47.980+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:48.296 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:47 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:47.980+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-04-25T04:35:17.345770+0000 front 2024-04-25T04:35:17.345779+0000 (oldest deadline 2024-04-25T04:35:43.146130+0000) 2024-04-25T04:35:48.296 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:47 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:47.980+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.112:6804 osd.4 since back 2024-04-25T04:35:25.347388+0000 front 2024-04-25T04:35:25.347407+0000 (oldest deadline 2024-04-25T04:35:47.647347+0000) 2024-04-25T04:35:49.397 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:48.956+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:49.397 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:48.956+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:49.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:48.956+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:49.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:48.956+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-04-25T04:35:17.345770+0000 front 2024-04-25T04:35:17.345779+0000 (oldest deadline 2024-04-25T04:35:43.146130+0000) 2024-04-25T04:35:49.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:48 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:48.956+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.112:6804 osd.4 since back 2024-04-25T04:35:25.347388+0000 front 2024-04-25T04:35:25.347407+0000 (oldest deadline 2024-04-25T04:35:47.647347+0000) 2024-04-25T04:35:50.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:49 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:49.929+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-04-25T04:34:53.442866+0000 front 2024-04-25T04:34:53.442789+0000 (oldest deadline 2024-04-25T04:35:19.342662+0000) 2024-04-25T04:35:50.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:49 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:49.929+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-04-25T04:35:00.443893+0000 front 2024-04-25T04:35:00.443845+0000 (oldest deadline 2024-04-25T04:35:25.743744+0000) 2024-04-25T04:35:50.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:49 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:49.929+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-04-25T04:35:06.244802+0000 front 2024-04-25T04:35:06.244814+0000 (oldest deadline 2024-04-25T04:35:31.544558+0000) 2024-04-25T04:35:50.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:49 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:49.929+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-04-25T04:35:17.345770+0000 front 2024-04-25T04:35:17.345779+0000 (oldest deadline 2024-04-25T04:35:43.146130+0000) 2024-04-25T04:35:50.398 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:49 smithi112 ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7[61760]: 2024-04-25T04:35:49.929+0000 7fdab2251700 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.112:6804 osd.4 since back 2024-04-25T04:35:25.347388+0000 front 2024-04-25T04:35:25.347407+0000 (oldest deadline 2024-04-25T04:35:47.647347+0000) 2024-04-25T04:35:51.547 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:51 smithi112 podman[77306]: 2024-04-25 04:35:51.271920759 +0000 UTC m=+5.541633562 container died d9186eec4bcc9594a0e33f9fe238db9310b915a88abedbd062ea7664873c1b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-25T04:35:51.897 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:51 smithi112 podman[77306]: 2024-04-25 04:35:51.703162383 +0000 UTC m=+5.972875191 container cleanup d9186eec4bcc9594a0e33f9fe238db9310b915a88abedbd062ea7664873c1b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:51.898 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:51 smithi112 bash[77306]: ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7 2024-04-25T04:35:52.298 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:51 smithi112 podman[77335]: 2024-04-25 04:35:51.89646676 +0000 UTC m=+0.621279644 container remove d9186eec4bcc9594a0e33f9fe238db9310b915a88abedbd062ea7664873c1b09 (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-25T04:35:52.610 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:52 smithi112 podman[77429]: 2024-04-25 04:35:52.199583081 +0000 UTC m=+0.018330802 image pull 3d0c579d3b62546f09ea44dc23ec09e0c00d5c2184768532a3c2aae78479c2f1 quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72 2024-04-25T04:35:52.610 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:52 smithi112 podman[77429]: 2024-04-25 04:35:52.330895259 +0000 UTC m=+0.149642969 container create 922a89e20a336d211c593b61512c1f45eea656fe4492eb8bd6283b3a90eb6e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-25T04:35:52.610 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:52 smithi112 podman[77429]: 2024-04-25 04:35:52.533358483 +0000 UTC m=+0.352106196 container init 922a89e20a336d211c593b61512c1f45eea656fe4492eb8bd6283b3a90eb6e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-25T04:35:52.610 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:52 smithi112 podman[77429]: 2024-04-25 04:35:52.536361268 +0000 UTC m=+0.355108983 container start 922a89e20a336d211c593b61512c1f45eea656fe4492eb8bd6283b3a90eb6e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-25T04:35:52.611 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:52 smithi112 podman[77429]: 2024-04-25 04:35:52.608275174 +0000 UTC m=+0.427022884 container attach 922a89e20a336d211c593b61512c1f45eea656fe4492eb8bd6283b3a90eb6e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:52.897 INFO:journalctl@ceph.osd.7.smithi112.stdout:Apr 25 04:35:52 smithi112 podman[77429]: 2024-04-25 04:35:52.824737211 +0000 UTC m=+0.643484921 container died 922a89e20a336d211c593b61512c1f45eea656fe4492eb8bd6283b3a90eb6e9e (image=quay.ceph.io/ceph-ci/ceph@sha256:53547c167dfd1636c2f97ada3832867c494b7ad3a6f460c09753bd0677c85e72, name=ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-24-1625-d6f5f4e, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-25T04:35:53.516 DEBUG:teuthology.orchestra.run.smithi112:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9a8d9f72-02b9-11ef-bc93-c7b262605968@osd.7.service' 2024-04-25T04:35:53.554 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-25T04:35:53.554 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-25T04:35:53.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-25T04:38:52.156 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-25T04:39:12.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T04:39:12.157 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-25T04:39:12.183 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-25T04:39:12.185 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738/remote/smithi078/crash 2024-04-25T04:39:12.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/crash -- . 2024-04-25T04:39:12.314 INFO:teuthology.orchestra.run.smithi078.stderr:tar: /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-25T04:39:12.314 INFO:teuthology.orchestra.run.smithi078.stderr:tar: Error is not recoverable: exiting now 2024-04-25T04:39:12.316 DEBUG:teuthology.misc:Transferring archived files from smithi112:/var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738/remote/smithi112/crash 2024-04-25T04:39:12.317 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/crash -- . 2024-04-25T04:39:12.359 INFO:teuthology.orchestra.run.smithi112.stderr:tar: /var/lib/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-25T04:39:12.359 INFO:teuthology.orchestra.run.smithi112.stderr:tar: Error is not recoverable: exiting now 2024-04-25T04:39:12.360 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-25T04:39:12.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-25T04:39:12.391 INFO:tasks.cephadm:Compressing logs... 2024-04-25T04:39:12.391 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-25T04:39:12.433 DEBUG:teuthology.orchestra.run.smithi112:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mon.smithi112.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:: No such file or directory 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mgr.smithi112.lvtlnw.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mon.smithi112.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mgr.smithi112.lvtlnw.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-25T04:39:12.459 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-25T04:39:12.461 INFO:teuthology.orchestra.run.smithi078.stderr:find: ‘/var/log/rbd-target-api’gzip: No such file or directory 2024-04-25T04:39:12.461 INFO:teuthology.orchestra.run.smithi078.stderr: -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mon.smithi078.log 2024-04-25T04:39:12.461 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.log 2024-04-25T04:39:12.461 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-25T04:39:12.461 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mon.smithi078.log: /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-25T04:39:12.467 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-25T04:39:12.467 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi078.nlrtle.log 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi078.nlrtle.log: gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mgr.smithi078.eugwik.log 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.0.log: 88.1% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi078.nlrtle.log.gz 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mgr.smithi078.eugwik.log: 87.1% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.cephadm.log: 85.8% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-25T04:39:12.468 INFO:teuthology.orchestra.run.smithi078.stderr: 91.6% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-25T04:39:12.469 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.6.log: 87.1% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-25T04:39:12.469 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi112.xminls.log 2024-04-25T04:39:12.469 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.cephadm.log: 85.8% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-25T04:39:12.469 INFO:teuthology.orchestra.run.smithi112.stderr: 91.7% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-25T04:39:12.469 INFO:teuthology.orchestra.run.smithi112.stderr:gzip -5 --verbose -- /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-25T04:39:12.469 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi112.xminls.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-25T04:39:12.469 INFO:teuthology.orchestra.run.smithi112.stderr: 87.8% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mds.foofs.smithi112.xminls.log.gz 2024-04-25T04:39:12.479 INFO:teuthology.orchestra.run.smithi112.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-25T04:39:12.479 INFO:teuthology.orchestra.run.smithi112.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-25T04:39:12.481 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-25T04:39:12.518 INFO:teuthology.orchestra.run.smithi078.stderr: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-25T04:39:12.524 INFO:teuthology.orchestra.run.smithi112.stderr: 89.7% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mgr.smithi112.lvtlnw.log.gz 2024-04-25T04:39:12.749 INFO:teuthology.orchestra.run.smithi078.stderr: 88.3% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mgr.smithi078.eugwik.log.gz 2024-04-25T04:39:12.771 INFO:teuthology.orchestra.run.smithi112.stderr: 92.6% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mon.smithi112.log.gz 2024-04-25T04:39:14.473 INFO:teuthology.orchestra.run.smithi078.stderr: 92.0% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-mon.smithi078.log.gz 2024-04-25T04:39:24.518 INFO:teuthology.orchestra.run.smithi078.stderr: 94.4% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-25T04:39:27.194 INFO:teuthology.orchestra.run.smithi112.stderr: 93.9% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-25T04:39:29.227 INFO:teuthology.orchestra.run.smithi078.stderr: 94.1% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-25T04:39:29.983 INFO:teuthology.orchestra.run.smithi112.stderr: 94.3% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-25T04:39:32.194 INFO:teuthology.orchestra.run.smithi078.stderr: 94.2% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-25T04:39:33.040 INFO:teuthology.orchestra.run.smithi112.stderr: 94.2% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-25T04:39:33.227 INFO:teuthology.orchestra.run.smithi078.stderr: 94.1% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-25T04:39:33.228 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-04-25T04:39:33.229 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m20.782s 2024-04-25T04:39:33.229 INFO:teuthology.orchestra.run.smithi078.stderr:user 1m8.913s 2024-04-25T04:39:33.229 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m2.674s 2024-04-25T04:39:38.847 INFO:teuthology.orchestra.run.smithi112.stderr: 94.1% -- replaced with /var/log/ceph/9a8d9f72-02b9-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-25T04:39:38.848 INFO:teuthology.orchestra.run.smithi112.stderr: 2024-04-25T04:39:38.849 INFO:teuthology.orchestra.run.smithi112.stderr:real 0m26.401s 2024-04-25T04:39:38.849 INFO:teuthology.orchestra.run.smithi112.stderr:user 1m16.493s 2024-04-25T04:39:38.849 INFO:teuthology.orchestra.run.smithi112.stderr:sys 0m3.024s 2024-04-25T04:39:38.849 INFO:tasks.cephadm:Archiving logs... 2024-04-25T04:39:38.849 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738/remote/smithi078/log 2024-04-25T04:39:38.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-25T04:39:48.766 DEBUG:teuthology.misc:Transferring archived files from smithi112:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738/remote/smithi112/log 2024-04-25T04:39:48.767 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-25T04:39:59.706 INFO:tasks.cephadm:Removing cluster... 2024-04-25T04:39:59.706 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 --force 2024-04-25T04:40:00.074 DEBUG:teuthology.orchestra.run.smithi112:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9a8d9f72-02b9-11ef-bc93-c7b262605968 --force 2024-04-25T04:40:00.464 INFO:tasks.cephadm:Removing cephadm ... 2024-04-25T04:40:00.465 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-25T04:40:00.480 DEBUG:teuthology.orchestra.run.smithi112:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-25T04:40:00.495 INFO:tasks.cephadm:Teardown complete 2024-04-25T04:40:00.495 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-25T04:40:00.505 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-25T04:40:00.505 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T04:40:00.523 DEBUG:teuthology.orchestra.run.smithi112:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-25T04:40:00.534 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-04-25T04:40:00.538 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T04:40:00.538 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-04-25T04:40:00.538 INFO:teuthology.orchestra.run.smithi078.stdout:^- hv01.front.sepia.ceph.com 2 8 377 82 +760us[ +781us] +/- 595ms 2024-04-25T04:40:00.539 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv02.front.sepia.ceph.com 3 6 377 18 -960us[ -938us] +/- 79ms 2024-04-25T04:40:00.539 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 15 +915us[ +915us] +/- 78ms 2024-04-25T04:40:00.539 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:40:00.547 INFO:teuthology.orchestra.run.smithi112.stderr:bash: line 1: ntpq: command not found 2024-04-25T04:40:00.552 INFO:teuthology.orchestra.run.smithi112.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-25T04:40:00.552 INFO:teuthology.orchestra.run.smithi112.stdout:=============================================================================== 2024-04-25T04:40:00.552 INFO:teuthology.orchestra.run.smithi112.stdout:^- hv01.front.sepia.ceph.com 2 6 377 20 +740us[ +781us] +/- 596ms 2024-04-25T04:40:00.552 INFO:teuthology.orchestra.run.smithi112.stdout:^* hv02.front.sepia.ceph.com 3 6 377 17 -950us[ -909us] +/- 80ms 2024-04-25T04:40:00.552 INFO:teuthology.orchestra.run.smithi112.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 17 +922us[ +922us] +/- 77ms 2024-04-25T04:40:00.552 INFO:teuthology.orchestra.run.smithi112.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-25T04:40:00.553 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-25T04:40:00.563 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-25T04:40:00.563 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-25T04:40:00.571 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-25T04:40:00.656 DEBUG:teuthology.orchestra.run.smithi112:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-25T04:40:00.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T04:40:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:40:00.754 DEBUG:teuthology.orchestra.run.smithi112:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-25T04:40:00.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:40:00.780 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-25T04:40:00.790 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-25T04:40:00.826 INFO:teuthology.task.internal:Duration was 2042.132285 seconds 2024-04-25T04:40:00.826 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-25T04:40:00.836 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-25T04:40:00.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-25T04:40:00.838 DEBUG:teuthology.orchestra.run.smithi112:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-25T04:40:00.871 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T04:40:00.874 INFO:teuthology.orchestra.run.smithi112.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-25T04:40:01.397 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-25T04:40:01.397 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-04-25T04:40:01.398 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-25T04:40:01.457 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi112.front.sepia.ceph.com 2024-04-25T04:40:01.458 DEBUG:teuthology.orchestra.run.smithi112:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-25T04:40:01.487 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-25T04:40:01.487 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-25T04:40:01.500 DEBUG:teuthology.orchestra.run.smithi112:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-25T04:40:01.552 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-25T04:40:01.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-25T04:40:01.565 DEBUG:teuthology.orchestra.run.smithi112:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-25T04:40:15.247 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-25T04:40:15.258 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-25T04:40:15.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-25T04:40:15.281 DEBUG:teuthology.orchestra.run.smithi112:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-25T04:40:15.310 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-25T04:40:15.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-25T04:40:15.324 DEBUG:teuthology.orchestra.run.smithi112:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-25T04:40:15.343 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-04-25T04:40:15.376 INFO:teuthology.orchestra.run.smithi112.stdout:kernel.core_pattern = core 2024-04-25T04:40:15.434 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-25T04:40:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:40:15.489 DEBUG:teuthology.orchestra.run.smithi112:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-25T04:40:15.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-25T04:40:15.504 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-25T04:40:15.515 INFO:teuthology.task.internal:Transferring archived files... 2024-04-25T04:40:15.516 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738/remote/smithi078 2024-04-25T04:40:15.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-25T04:40:15.588 DEBUG:teuthology.misc:Transferring archived files from smithi112:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-25_03:53:55-orch:cephadm-wip-adk-testing-2024-04-24-1625-distro-default-smithi/7672738/remote/smithi112 2024-04-25T04:40:15.588 DEBUG:teuthology.orchestra.run.smithi112:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-25T04:40:15.627 INFO:teuthology.task.internal:Removing archive directory... 2024-04-25T04:40:15.627 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-25T04:40:15.630 DEBUG:teuthology.orchestra.run.smithi112:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-25T04:40:15.676 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-25T04:40:15.689 INFO:teuthology.task.internal:Not uploading archives. 2024-04-25T04:40:15.689 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-25T04:40:15.699 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-25T04:40:15.699 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-25T04:40:15.702 DEBUG:teuthology.orchestra.run.smithi112:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-25T04:40:15.714 INFO:teuthology.orchestra.run.smithi078.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 25 04:40 /home/ubuntu/cephtest 2024-04-25T04:40:15.733 INFO:teuthology.orchestra.run.smithi112.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 25 04:40 /home/ubuntu/cephtest 2024-04-25T04:40:15.733 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-25T04:40:15.745 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-25T04:40:15.770 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2042.1322853565216 owner: scheduled_adking@teuthology success: true 2024-04-25T04:40:15.770 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-25T04:40:15.850 INFO:teuthology.run:pass